diff --git a/build.sbt b/build.sbt index bbdb6ca..0823004 100644 --- a/build.sbt +++ b/build.sbt @@ -4,8 +4,16 @@ Global / onChangedBuildSource := ReloadOnSourceChanges inThisBuild( Seq( version := "0.1", - scalaVersion := "2.13.5", - scalacOptions ++= "-Ymacro-annotations" :: "-language:experimental.macros" :: Nil + scalaVersion := "2.13.6", + scalacOptions ++= Seq( + "-Ymacro-annotations", + "-language:experimental.macros", + //"-Xlog-implicits", + "-Ymacro-debug-lite", + // for nicer error messages + //"-Vimplicits", + "-Vtype-diffs" + ) ) ) @@ -16,12 +24,19 @@ lazy val `scala-macros` = (project in file("scala-macros")) .settings( libraryDependencies ++= Seq( scalaReflect % scalaVersion.value, - scalaPbRuntime + scalaPbRuntime, + beamSdksJavaCore, + beamRunnersDirectJava ) ) lazy val `scala-macros-usage` = (project in file("scala-macros-usage")) .dependsOn(`scala-macros`) .settings( - libraryDependencies ++= scalaPbRuntime :: Nil + libraryDependencies ++= Seq( + scalaPbRuntime, + beamSdksJavaCore, + beamRunnersDirectJava, + scalaTest + ) ) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index accb36b..055b1e2 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -3,4 +3,10 @@ import sbt._ object Dependencies { val scalaReflect = "org.scala-lang" % "scala-reflect" val scalaPbRuntime = "com.thesamet.scalapb" %% "scalapb-runtime" % scalapb.compiler.Version.scalapbVersion + + val beamVersion = "2.28.0" + val beamSdksJavaCore = "org.apache.beam" % "beam-sdks-java-core" % beamVersion + val beamRunnersDirectJava = "org.apache.beam" % "beam-runners-direct-java" % beamVersion % Runtime + + val scalaTest = "org.scalatest" %% "scalatest" % "3.2.7" % Test } diff --git a/scala-macros-usage/src/main/scala/com/github/fpopic/scalamacros/AnnotationMacroUsage.scala b/scala-macros-usage/src/main/scala/com/github/fpopic/scalamacros/AnnotationMacroUsage.scala deleted file mode 100644 index e699bbc..0000000 --- a/scala-macros-usage/src/main/scala/com/github/fpopic/scalamacros/AnnotationMacroUsage.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.github.fpopic.scalamacros - -//@identity -object MyObject { - - @identity(i = 123) - case class MyCaseClass(x: Int, y: List[String]) - - // @identity - - val myVal = 1 - -} - -//@identity -class MyClass(_c: Int) { - - val c: Int = _c - -} - -@identity(i = 456) -case class MyCaseClass2(x: Int, y: List[String]) diff --git a/scala-macros-usage/src/main/scala/com/github/fpopic/scalamacros/DefMacroUsage.scala b/scala-macros-usage/src/main/scala/com/github/fpopic/scalamacros/DefMacroUsage.scala deleted file mode 100644 index 60c71d8..0000000 --- a/scala-macros-usage/src/main/scala/com/github/fpopic/scalamacros/DefMacroUsage.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.github.fpopic.scalamacros - -case class N(j: Int, @identity(i = 5) k: Int) - -case class A(i: Int, s: Some[Int], l: List[Int], n: N) - -object DefMacroUsage { - - def main(args: Array[String]): Unit = { - - import ToMap._ - - //val x = ToMap[A].toMap(A(1, Some(1), List(1), N(1, 1))) - - val a: Map[String, Any] = - mapify( - A( - i = 1, - s = Some(2), - l = List(3, 4), - n = N( - j = 5, - k = 7 - ) - ) - ) - - println(a) - - } -} diff --git a/scala-macros-usage/src/main/scala/com/github/fpopic/scalamacros/Person.scala b/scala-macros-usage/src/main/scala/com/github/fpopic/scalamacros/Person.scala deleted file mode 100644 index 1dc17e0..0000000 --- a/scala-macros-usage/src/main/scala/com/github/fpopic/scalamacros/Person.scala +++ /dev/null @@ -1,3 +0,0 @@ -package com.github.fpopic.scalamacros - -case class Person(firstName: String, lastName: String, age: Int) diff --git a/scala-macros-usage/src/test/scala/com/github/fpopic/scalamacros/beam/DefMacroCoderSpec.scala b/scala-macros-usage/src/test/scala/com/github/fpopic/scalamacros/beam/DefMacroCoderSpec.scala new file mode 100644 index 0000000..4e12599 --- /dev/null +++ b/scala-macros-usage/src/test/scala/com/github/fpopic/scalamacros/beam/DefMacroCoderSpec.scala @@ -0,0 +1,49 @@ +package com.github.fpopic.scalamacros.beam + +import org.apache.beam.sdk.coders.Coder +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.io.{ByteArrayInputStream, ByteArrayOutputStream} + +case class Pojo(s: String, i: Int, l: List[Int], s2: String) +case class ContainsNested(i: Int, pojo: Pojo) + +class DefMacroCoderSpec extends AnyFlatSpec with Matchers { + + behavior of "DefMacroCoder" + + it should "generate a coder and serialize/deserialize properly the value." in { + import DefMacroCoder.{intCoder, stringCoder, listCoder} + val coder: Coder[Pojo] = DefMacroCoder.productCoder[Pojo] + + val pojo = Pojo(s = "4", i = 1, l = List(2, 3), s2 = "5") + + val encoded: Array[Byte] = { + val os = new ByteArrayOutputStream() + coder.encode(pojo, os) + os.toByteArray + } + val decoded: Pojo = coder.decode(new ByteArrayInputStream(encoded)) + + pojo shouldBe decoded + } + + it should "generate a coder for nested case class and serialize/deserialize properly the value." in { + import DefMacroCoder.{intCoder, stringCoder, listCoder} + implicit val pojoCoder: Coder[Pojo] = DefMacroCoder.productCoder[Pojo] + val coder: Coder[ContainsNested] = DefMacroCoder.productCoder[ContainsNested] + + val pojo = ContainsNested(i = 7, pojo = Pojo(s = "4", i = 1, l = List(2, 3), s2 = "5")) + + val encoded: Array[Byte] = { + val os = new ByteArrayOutputStream() + coder.encode(pojo, os) + os.toByteArray + } + val decoded: ContainsNested = coder.decode(new ByteArrayInputStream(encoded)) + + pojo shouldBe decoded + } + +} diff --git a/scala-macros/src/main/scala/com/github/fpopic/scalamacros/AnnotationMacroIdentityExample.scala b/scala-macros/src/main/scala/com/github/fpopic/scalamacros/AnnotationMacroIdentityExample.scala deleted file mode 100644 index 976c605..0000000 --- a/scala-macros/src/main/scala/com/github/fpopic/scalamacros/AnnotationMacroIdentityExample.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.github.fpopic.scalamacros - -import scala.annotation.{StaticAnnotation, compileTimeOnly} -import scala.reflect.api.Trees -import scala.reflect.macros.blackbox - -// Quasiquotes Syntax: -// https://docs.scala-lang.org/overviews/quasiquotes/syntax-summary.html - -// 1. Provide macro impl. -object Macros { - - // annottees -> all thing where we put @identity - def impl(c: blackbox.Context)(annottees: c.Tree*): c.Tree = { - import c.universe._ - val helper = new MacrosHelper[c.type](c) - - println("\nAnnotationMacro:") - - // To extract annotation parameter - val i: Int = c.prefix.tree match { - case q @ q"new identity(i=$i)" => - helper.evalTree[Int](i) - case _ => 0 - } - - println(s"Parameter i:$i") - - val inputs = annottees.toList - - val (annottee, expandees) = inputs match { - case (param: ValDef) :: (rest @ _ :: _) => - (param, rest) - - case (param: TypeDef) :: (rest @ _ :: _) => - (param, rest) - - case _ => - (EmptyTree, inputs) - } - - println("XX" + (annottee, expandees)) - - Block(expandees, Literal(Constant(()))) - } - -} - -// 0. Define anotation -@compileTimeOnly("enable macro paradise to expand macro annotations") -class identity(i: Int) extends StaticAnnotation { - def macroTransform(annottees: Any*): Any = macro Macros.impl -} diff --git a/scala-macros/src/main/scala/com/github/fpopic/scalamacros/DefMacroToMapExample.scala b/scala-macros/src/main/scala/com/github/fpopic/scalamacros/DefMacroToMapExample.scala deleted file mode 100644 index d847885..0000000 --- a/scala-macros/src/main/scala/com/github/fpopic/scalamacros/DefMacroToMapExample.scala +++ /dev/null @@ -1,93 +0,0 @@ -package com.github.fpopic.scalamacros - -import scala.reflect.macros.blackbox - -// 0. Define Type Class -trait ToMap[T] { - - def toMap(t: T): Map[String, Any] - -} - -object ToMap extends ToMapLowPriorityImplicits { - // 2. Implicit method that triggers the macro - - // HighPriorityImplicits - // I can now only make implicits for whole case class but not for fields, - // for them i need a new type class or in pattern matching in mapEntries code add cases - - // LowPriorityMacros - implicit def caseClassMappable[T]: ToMap[T] = macro materializeMappableImpl[T] - - // 1. Caller initiates type class implicits resolution - def mapify[T](t: T)(implicit m: ToMap[T]): Map[String, Any] = m.toMap(t) -} - -trait ToMapLowPriorityImplicits { - - // 3. Macro that generates for any case class ToMap implementation - def materializeMappableImpl[T: c.WeakTypeTag](c: blackbox.Context): c.Expr[ToMap[T]] = { - import c.universe._ - val helper = new MacrosHelper[c.type](c) - - val tpe = weakTypeOf[T] - println(s"\nDefMacro: $tpe") - - // For each constructor field genrate tree that repr. tuple ("name" -> value) - val mapEntries: Seq[c.Tree] = - helper.getPrimaryConstructorMembers(tpe).map { field => - val fName = field.name.decodedName.toString - val fTerm = field.name.toTermName - - field.annotations.foreach { a => - println("Annotations in def macro: " + a.tree.tpe.toString) - a - } - - // doesn't work with tType.decl(field.name).typeSignature - val fType = field.typeSignature - fType match { - case t if t =:= weakTypeOf[Int] => - println(s"$fName : $fType") - q"$fName -> (t.$fTerm + 100)" - case t if t =:= weakTypeOf[Some[Int]] => - println(s"$fName : $fType") - q"$fName -> t.$fTerm.get" - case t if t <:< weakTypeOf[Option[_]] => - // FIXME - q"""if ($fTerm.isDefined) - $fName -> $fTerm.get - else "" - """ - case t if t =:= weakTypeOf[List[Int]] => - println(s"$fName : $fType") - q"$fName -> t.$fTerm" - // in case field is nested case class - case n if n.baseClasses.contains(weakTypeOf[Product].typeSymbol) => - println(s"$fName : $fType") - q"$fName -> mapify(t.$fTerm)" - case t => - try { - // default ones - } catch { - case _: Exception => c.abort(c.enclosingPosition, s"Type $t is not supported.") - } - q"" - } - } - - val ret = - q"""new ToMap[$tpe] { - def toMap(t: $tpe): Map[String, Any] = Map(..$mapEntries) - } - """ - - println(s"Ret: $ret") - - c.Expr[ToMap[T]](ret) - } - -} - -// check byte code -// javap -c scala-macros-usage/target/scala-2.13.0-M3/classes/com/github/fpopic/scalamacros/A\$.class diff --git a/scala-macros/src/main/scala/com/github/fpopic/scalamacros/beam/DefMacroCoder.scala b/scala-macros/src/main/scala/com/github/fpopic/scalamacros/beam/DefMacroCoder.scala new file mode 100644 index 0000000..8a949fe --- /dev/null +++ b/scala-macros/src/main/scala/com/github/fpopic/scalamacros/beam/DefMacroCoder.scala @@ -0,0 +1,118 @@ +package com.github.fpopic.scalamacros.beam + +import com.github.fpopic.scalamacros.MacrosHelper +import org.apache.beam.sdk.coders.{Coder, ListCoder, StringUtf8Coder, VarIntCoder} + +import java.io.{InputStream, OutputStream} +import java.util +import java.util.Collections +import scala.jdk.CollectionConverters._ + +object DefMacroCoder extends LowPriorityMacros { + + // Coder[T] will be my typeclass + def of[T](implicit c: Coder[T]): Coder[T] = c + + /* HighPriority */ + // some typeclass implementations + implicit val stringCoder: Coder[String] = StringUtf8Coder.of() + + implicit val intCoder: Coder[Int] = new Coder[Int] { + + private val baseCoder = VarIntCoder.of() + + override def encode(i: Int, os: OutputStream): Unit = baseCoder.encode(i, os) + + override def decode(is: InputStream): Int = baseCoder.decode(is) + + override def getCoderArguments: util.List[_ <: Coder[_]] = Collections.emptyList() + + override def verifyDeterministic(): Unit = baseCoder.verifyDeterministic() + } + + implicit def listCoder[T](implicit c: Coder[T]): Coder[List[T]] = new Coder[List[T]] { + + private val baseCoder = ListCoder.of(c) + + override def encode(l: List[T], os: OutputStream): Unit = baseCoder.encode(l.asJava, os) + + override def decode(is: InputStream): List[T] = baseCoder.decode(is).asScala.toList // fix it + + override def getCoderArguments: util.List[_ <: Coder[_]] = Collections.emptyList() + + override def verifyDeterministic(): Unit = baseCoder.verifyDeterministic() + } + + /* LowPriority */ + implicit def productCoder[P <: Product]: Coder[P] = macro materializeProductCoder[P] +} + +trait LowPriorityMacros { + + import scala.reflect.macros.blackbox + + def materializeProductCoder[P: c.WeakTypeTag](c: blackbox.Context): c.Expr[Coder[P]] = { + import c.universe._ + val tpe = c.weakTypeOf[P] + val helper = new MacrosHelper[c.type](c) + + val expressions = + helper.getPrimaryConstructorMembers(tpe).map { field => + val fieldTerm = field.asTerm.name // e.g. value.s (for now just s) + val fieldType = field.typeSignature.finalResultType // e.g. String + + val fieldCoderName = c.freshName(TermName("coder")) // e.g. give friendly name coder$... + val fieldCoderInstance = // e.g. finds instance of Coder[String] + c.typecheck( + tree = q"""_root_.scala.Predef.implicitly[org.apache.beam.sdk.coders.Coder[${fieldType}]]""", + silent = false + ) + + val fieldCoderExpression = + q"private val ${fieldCoderName}: org.apache.beam.sdk.coders.Coder[${fieldType}] = ${fieldCoderInstance}" + + val fieldEncodeExpression = + q"${fieldCoderName}.encode(value.${fieldTerm}, os)" // replace with full relative name (with dots) instead of value + + val fieldDecodeExpression = + q"${field.asTerm} = ${fieldCoderName}.decode(is)" + + (fieldCoderExpression, fieldEncodeExpression, fieldDecodeExpression) + } + + val fieldCodersExpression = expressions.map(_._1).distinct + val coderEncodeExpresions = expressions.map(_._2) + val coderDecodeExpresions = expressions.map(_._3) + + val coderExpression = + q"""{ + new org.apache.beam.sdk.coders.Coder[${tpe}] { + + {import ${c.prefix}._} + + ..${fieldCodersExpression} + + override def encode(value: ${tpe}, os: java.io.OutputStream): _root_.scala.Unit = { + ..${coderEncodeExpresions} + } + + override def decode(is: java.io.InputStream): ${tpe} = { + new ${tpe}( + ..${coderDecodeExpresions} + ) + } + + override def getCoderArguments: java.util.List[_ <: org.apache.beam.sdk.coders.Coder[_]] = { + java.util.Collections.emptyList + } + + override def verifyDeterministic(): _root_.scala.Unit = () + } + } + """ + + val ret = coderExpression + c.Expr[Coder[P]](ret) + } + +} diff --git a/scala-macros/src/main/scala/com/github/fpopic/scalamacros/beam/PROBLEM.md b/scala-macros/src/main/scala/com/github/fpopic/scalamacros/beam/PROBLEM.md new file mode 100644 index 0000000..c661161 --- /dev/null +++ b/scala-macros/src/main/scala/com/github/fpopic/scalamacros/beam/PROBLEM.md @@ -0,0 +1,47 @@ +For the input: + +```scala +import com.github.fpopic.scalamacros.beam.DefMacroCoder +import com.github.fpopic.scalamacros.beam.DefMacroCoder.{intCoder, stringCoder, listCoder} + +val coder: Coder[Pojo] = DefMacroCoder.productCoder[Pojo] +``` + +The generated output should be: + +```scala +import com.github.fpopic.scalamacros.Pojo +import org.apache.beam.sdk.coders.Coder +import java.io.{ByteArrayInputStream, ByteArrayOutputStream} +import java.util + +new Coder[Pojo] { + + import com.github.fpopic.scalamacros.beam.DefMacroCoder.{ + intCoder, + stringCoder, + listCoder + } + + override def encode(value: Pojo, os: OutputStream): Unit = { + stringCoder.encode(value.s, os) + intCoder.encode(value.i, os) + listCoder(intCoder).encode(value.l, os) + } + + override def decode(is: InputStream): Pojo = { + Pojo( + s = stringCoder.decode(is), + i = intCoder.decode(is), + l = listCoder(intCoder).decode(is) + ) + } + + override def getCoderArguments: util.List[_ <: Coder[_]] = { + Collections.emptyList() + } + + override def verifyDeterministic(): Unit = () +} + +``` diff --git a/scala-macros/src/main/scala/com/github/fpopic/scalamacros/scalapb/DefMacroMessageSerializerExample.scala b/scala-macros/src/main/scala/com/github/fpopic/scalamacros/scalapb/DefMacroMessageSerializerExample.scala deleted file mode 100644 index f270fbc..0000000 --- a/scala-macros/src/main/scala/com/github/fpopic/scalamacros/scalapb/DefMacroMessageSerializerExample.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.github.fpopic.scalamacros.scalapb - -import com.google.protobuf.CodedOutputStream - -trait MessageSerializer[T] { - def toByteArray(message: T): Array[Byte] = { - val size = serializedSize(message) - val bytes = new Array[Byte](size) - val output = CodedOutputStream.newInstance(bytes) - writeTo(output, message) - bytes - } - - def writeTo(output: CodedOutputStream, message: T): Unit - - def serializedSize(message: T): Int -} - -object MessageSerializer { - // apply method inside companion object - // instead of making a new object it runs an implicit search and returns an instance that it founds - def apply[T](implicit ms: MessageSerializer[T]): MessageSerializer[T] = ms - - implicit val intSerializer: MessageSerializer[Int] = new MessageSerializer[Int] { - def writeTo(output: CodedOutputStream, i: Int): Unit = output.writeInt32(1, i) - - def serializedSize(message: Int): Int = 4 - } - - implicit val boolSerializer: MessageSerializer[Boolean] = new MessageSerializer[Boolean] { - def writeTo(output: CodedOutputStream, b: Boolean): Unit = output.writeBool(2, b) - - def serializedSize(b: Boolean): Int = 1 - } - - // todo with shapeless I can have field serializer - // todo with macros I could have full message serializer -}