From 0b1e1269ba112dc76eb4e2d67a003d1559c03fd2 Mon Sep 17 00:00:00 2001 From: Filipe Regadas Date: Thu, 21 Nov 2019 17:06:25 -0500 Subject: [PATCH] Rework plugin (#69) --- .scalafmt.conf | 28 ++ NOTICE | 2 +- README.md | 2 +- build.sbt | 2 +- src/main/resources/META-INF/plugin.xml | 9 +- .../spotify/scio/AnnotationTypeInjector.scala | 174 ++++++++++ .../com/spotify/scio/AvroTypeInjector.scala | 110 ++++++ .../spotify/scio/BigQueryTypeInjector.scala | 144 ++++++++ .../scala/com/spotify/scio/ScioInjector.scala | 328 ------------------ .../com/spotify/scio/ScioInjectorTest.scala | 99 +++--- version.sbt | 2 +- 11 files changed, 515 insertions(+), 385 deletions(-) create mode 100644 src/main/scala/com/spotify/scio/AnnotationTypeInjector.scala create mode 100644 src/main/scala/com/spotify/scio/AvroTypeInjector.scala create mode 100644 src/main/scala/com/spotify/scio/BigQueryTypeInjector.scala delete mode 100644 src/main/scala/com/spotify/scio/ScioInjector.scala diff --git a/.scalafmt.conf b/.scalafmt.conf index 6a86bc7..a0b4e21 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -1 +1,29 @@ version = "2.2.2" +maxColumn = 100 + +binPack.literalArgumentLists = true + +continuationIndent { + callSite = 2 + defnSite = 2 +} + +newlines { + afterImplicitKWInVerticalMultiline = true + beforeImplicitKWInVerticalMultiline = true + sometimesBeforeColonInMethodReturnType = true +} + +docstrings = JavaDoc + +project.git = false + +rewrite { + rules = [PreferCurlyFors, RedundantBraces, RedundantParens, SortImports] + redundantBraces.maxLines = 1 +} + +project.excludeFilters = [ + "MultiJoin.scala", + "TupleCoders.scala" +] diff --git a/NOTICE b/NOTICE index 0e20757..c753c7b 100644 --- a/NOTICE +++ b/NOTICE @@ -1,2 +1,2 @@ Scio IDEA Plugin -Copyright 2016 Spotify AB +Copyright 2019 Spotify AB diff --git a/README.md b/README.md index 3b08f6b..1ac2f7a 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,6 @@ If there is error level message logged, it will show up in IntelliJ Event Log. # License -Copyright 2016 Spotify AB. +Copyright 2019 Spotify AB. Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 diff --git a/build.sbt b/build.sbt index 5e7fedd..2554b63 100644 --- a/build.sbt +++ b/build.sbt @@ -1,5 +1,5 @@ /* - * Copyright 2017 Spotify AB. + * Copyright 2019 Spotify AB. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/main/resources/META-INF/plugin.xml b/src/main/resources/META-INF/plugin.xml index d13a578..5a4d974 100644 --- a/src/main/resources/META-INF/plugin.xml +++ b/src/main/resources/META-INF/plugin.xml @@ -1,17 +1,18 @@ com.spotify.scio-idea Scio IDEA - 0.1.18 + 0.1.19 Spotify IntelliJ IDEA plugin for Scio - https://github.com/spotify/scio - + com.intellij.modules.java org.intellij.scala - + + - + \ No newline at end of file diff --git a/src/main/scala/com/spotify/scio/AnnotationTypeInjector.scala b/src/main/scala/com/spotify/scio/AnnotationTypeInjector.scala new file mode 100644 index 0000000..366d4ae --- /dev/null +++ b/src/main/scala/com/spotify/scio/AnnotationTypeInjector.scala @@ -0,0 +1,174 @@ +/* + * Copyright 2019 Spotify AB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.spotify.scio + +import java.io.File +import java.nio.charset.Charset +import java.nio.file.{Paths, Files => JFiles} + +import com.google.common.base.Charsets +import com.google.common.hash.Hashing +import com.google.common.io.Files +import com.intellij.notification.{Notification, NotificationType, Notifications} +import com.intellij.openapi.diagnostic.Logger +import com.intellij.psi.PsiElement +import org.jetbrains.plugins.scala.lang.psi.api.toplevel.typedef.{ScClass, ScTypeDefinition} +import org.jetbrains.plugins.scala.lang.psi.impl.toplevel.typedef.SyntheticMembersInjector + +import scala.collection.mutable + +object AnnotationTypeInjector { + private val Log = Logger.getInstance(classOf[AnnotationTypeInjector]) + private val CaseClassArgs = """case\s+class\s+[^(]+\((.*)\).*""".r + private val TypeArg = """[a-zA-Z0-9]+\s*:\s*[a-zA-Z0-9._]+([\[(](.*?)[)\]]+)?""".r + private val AlertEveryMissedXInvocations = 5 + + def getApplyPropsSignature(caseClasses: Option[String]): Seq[String] = + getConstructorProps(caseClasses) + .map(_.props) + .getOrElse(Seq.empty) + + def getConstructorProps(caseClasses: Option[String]): Option[ConstructorProps] = + caseClasses.collect { + case CaseClassArgs(params) => ConstructorProps(TypeArg.findAllIn(params).toSeq) + } + + def getUnapplyReturnTypes(caseClasses: Option[String]): Seq[String] = + getConstructorProps(caseClasses).map(_.types).getOrElse(Seq.empty) + + def getTupledMethod(returnClassName: String, caseClasses: Option[String]): String = { + val maybeTupledMethod = getConstructorProps(caseClasses).map { + case cp: ConstructorProps if (2 to 22).contains(cp.types.size) => + s"def tupled: _root_.scala.Function1[( ${cp.types.mkString(" , ")} ), $returnClassName ] = ???" + case _ => + "" + } + + maybeTupledMethod.getOrElse("") + } + + final case class ConstructorProps(props: Seq[String]) { + val types: Seq[String] = props.map(_.split(" : ")(1).trim) + } + + /** + * Finds BigQuery cache file, must be in sync with Scio implementation, otherwise plugin will + * not be able to find scala files. + */ + private def file(filename: String): Option[File] = { + val sysPropOverride = + Seq("generated.class.cache.directory", "bigquery.class.cache.directory") + .flatMap(sys.props.get) + .headOption + .map(Paths.get(_).resolve(filename)) + + val resolved = sysPropOverride.getOrElse { + val oldBqPath = Paths + .get(sys.props("java.io.tmpdir")) + .resolve("bigquery-classes") + .resolve(filename) + val newBqPath = Paths + .get(sys.props("java.io.tmpdir")) + .resolve(sys.props("user.name")) + .resolve("bigquery-classes") + .resolve(filename) + val path = Paths + .get(sys.props("java.io.tmpdir")) + .resolve(sys.props("user.name")) + .resolve("generated-classes") + .resolve(filename) + + Seq(path, newBqPath, oldBqPath).find(JFiles.exists(_)).getOrElse(path) + } + + val file = resolved.toFile + if (file.exists()) { + Log.debug(s"Found $resolved") + Some(file) + } else { + Log.warn(s"missing file: $resolved") + None + } + } + + /** + * Computes hash for macro - the hash must be consistent with hash implementation in Scio. + */ + def hash(owner: String, srcFile: String): String = + Hashing + .murmur3_32() + .newHasher() + .putString(owner, Charsets.UTF_8) + .putString(srcFile, Charsets.UTF_8) + .hash() + .toString +} + +trait AnnotationTypeInjector extends SyntheticMembersInjector { + import AnnotationTypeInjector._ + + private[this] val classMissed = + mutable.HashMap.empty[String, Int].withDefaultValue(0) + + protected def generatedCaseClasses(source: String, c: ScClass) = { + // For some reason sometimes [[getVirtualFile]] returns null, use Option. I don't know why. + val fileName = + Option(c.asInstanceOf[PsiElement].getContainingFile.getVirtualFile) + // wrap VirtualFile to java.io.File to use OS file separator + .map(vf => new File(vf.getCanonicalPath).getCanonicalPath) + + val file = for { + psiElementPath <- fileName + hash <- Some(hash(source, psiElementPath)) + cf <- classFile(c, hash) + } yield cf + + file.fold(Seq.empty[String]) { f => + import collection.JavaConverters._ + Files + .readLines(f, Charset.defaultCharset()) + .asScala + .filter(_.contains("case class")) + } + } + + protected def classFile(klass: ScClass, hash: String): Option[java.io.File] = { + val filename = s"${klass.name}-$hash.scala" + file(filename) match { + case f: Some[File] => + classMissed(filename) = 0 + f + case none => + classMissed(filename) += 1 + val errorMessage = + "Scio plugin could not find scala files for code completion. Please (re)compile the project." + if (classMissed(filename) >= AlertEveryMissedXInvocations) { + // reset counter + classMissed(filename) = 0 + val notification = new Notification( + "ScioIDEA", + "Scio Plugin", + errorMessage, + NotificationType.ERROR + ) + Notifications.Bus.notify(notification) + } + none + } + } +} diff --git a/src/main/scala/com/spotify/scio/AvroTypeInjector.scala b/src/main/scala/com/spotify/scio/AvroTypeInjector.scala new file mode 100644 index 0000000..4a464ce --- /dev/null +++ b/src/main/scala/com/spotify/scio/AvroTypeInjector.scala @@ -0,0 +1,110 @@ +/* + * Copyright 2019 Spotify AB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.spotify.scio + +import java.io.File +import java.nio.charset.Charset +import java.nio.file.{Paths, Files => JFiles} + +import com.google.common.base.Charsets +import com.google.common.hash.Hashing +import com.google.common.io.Files +import com.intellij.notification.{Notification, NotificationType, Notifications} +import com.intellij.openapi.diagnostic.Logger +import com.intellij.psi.PsiElement +import org.jetbrains.plugins.scala.lang.psi.api.toplevel.typedef.{ScClass, ScTypeDefinition} +import org.jetbrains.plugins.scala.lang.psi.impl.toplevel.typedef.SyntheticMembersInjector + +import scala.collection.mutable + +object AvroTypeInjector { + private val Log = Logger.getInstance(classOf[AvroTypeInjector]) + + private val AvroTNamespace = "AvroType" + private val AvroAnnotations = Seq( + s"$AvroTNamespace.fromSchema", + s"$AvroTNamespace.fromPath", + s"$AvroTNamespace.toSchema" + ) + private val CaseClassSuper = + "_root_.com.spotify.scio.avro.types.AvroType.HasAvroAnnotation" + + private def avroAnnotation(sc: ScClass): Option[String] = + sc.annotations + .map(_.getText) + .find(t => AvroAnnotations.exists(t.contains)) +} + +final class AvroTypeInjector extends AnnotationTypeInjector { + import AvroTypeInjector._ + import AnnotationTypeInjector._ + + override def needsCompanionObject(source: ScTypeDefinition): Boolean = false + + override def injectFunctions(source: ScTypeDefinition): Seq[String] = + source match { + case c: ScClass if avroAnnotation(c).isDefined => + val parent = c.containingClass.getQualifiedName.init + val caseClasses = generatedCaseClasses(parent, c).find { c => + c.contains(CaseClassSuper) + } + getApplyPropsSignature(caseClasses).map(v => s"def $v = ???") + case _ => Seq.empty + } + + override def injectSupers(source: ScTypeDefinition): Seq[String] = + source match { + case c: ScClass if avroAnnotation(c).isDefined => Seq(CaseClassSuper) + case _ => Seq.empty + } + + /** + * Main method of the plugin. Injects syntactic inner members like case classes and companion + * objects, makes IntelliJ happy about BigQuery macros. Assumes macro is enclosed within + * class/object. + */ + override def injectInners(source: ScTypeDefinition): Seq[String] = { + source.extendsBlock.members + .collect { + case c: ScClass if avroAnnotation(c).isDefined => + val caseClasses = + generatedCaseClasses(source.getQualifiedName.init, c).find { c => + c.contains(CaseClassSuper) + } + (c, caseClasses) + } + .collect { + case (c, caseClasses) if caseClasses.nonEmpty => + val tupledMethod = getTupledMethod(c.getName, caseClasses) + val applyPropsSignature = + getApplyPropsSignature(caseClasses).mkString(",") + val unapplyReturnTypes = + getUnapplyReturnTypes(caseClasses).mkString(",") + + s"""|object ${c.getName} { + | def apply( $applyPropsSignature ): ${c.getName} = ??? + | def unapply(x$$0: ${c.getName}): _root_.scala.Option[($unapplyReturnTypes)] = ??? + | def fromGenericRecord: _root_.scala.Function1[_root_.org.apache.avro.generic.GenericRecord, ${c.getName} ] = ??? + | def toGenericRecord: _root_.scala.Function1[ ${c.getName}, _root_.org.apache.avro.generic.GenericRecord] = ??? + | def schema: _root_.org.apache.avro.Schema = ??? + | def toPrettyString(indent: Int = 0): String = ??? + | $tupledMethod + |}""".stripMargin + } + } +} diff --git a/src/main/scala/com/spotify/scio/BigQueryTypeInjector.scala b/src/main/scala/com/spotify/scio/BigQueryTypeInjector.scala new file mode 100644 index 0000000..f990e5e --- /dev/null +++ b/src/main/scala/com/spotify/scio/BigQueryTypeInjector.scala @@ -0,0 +1,144 @@ +/* + * Copyright 2019 Spotify AB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.spotify.scio + +import java.io.File +import java.nio.charset.Charset +import java.nio.file.{Paths, Files => JFiles} + +import com.google.common.base.Charsets +import com.google.common.hash.Hashing +import com.google.common.io.Files +import com.intellij.notification.{Notification, NotificationType, Notifications} +import com.intellij.openapi.diagnostic.Logger +import com.intellij.psi.PsiElement +import org.jetbrains.plugins.scala.lang.psi.api.toplevel.typedef.{ScClass, ScTypeDefinition} +import org.jetbrains.plugins.scala.lang.psi.impl.toplevel.typedef.SyntheticMembersInjector + +import scala.collection.mutable + +object BigQueryTypeInjector { + private val Log = Logger.getInstance(classOf[BigQueryTypeInjector]) + + // Could not find a way to get fully qualified annotation names + // even tho there is API, it does not return the annotations. + // For now stick with relative annotation names. + private val BQTNamespace = "BigQueryType" + private val FromQuery = s"$BQTNamespace.fromQuery" + private val FromTable = s"$BQTNamespace.fromTable" + private val FromStorage = s"$BQTNamespace.fromStorage" + private val BigQueryAnnotations = Seq( + FromQuery, + FromTable, + FromStorage, + s"$BQTNamespace.fromSchema", + s"$BQTNamespace.toTable" + ) + + private val CaseClassSuper = + "_root_.com.spotify.scio.bigquery.types.BigQueryType.HasAnnotation" + + private def bqAnnotation(sc: ScClass): Option[String] = + sc.annotations + .map(_.getText) + .find(t => BigQueryAnnotations.exists(t.contains)) + + private def fetchExtraBQTypeCompanionMethods(source: ScTypeDefinition, c: ScClass): String = { + val annotation = bqAnnotation(c).getOrElse("") + Log.debug(s"Found $annotation in ${source.getQualifiedNameForDebugger}") + + annotation match { + case a if a.contains(FromQuery) => + "def query: _root_.java.lang.String = ???" + case a if a.contains(FromTable) => + "def table: _root_.java.lang.String = ???" + case a if a.contains(FromStorage) => + """ + |def table: _root_.java.lang.String = ??? + |def selectedFields: _root_.scala.List[_root_.java.lang.String] = ??? + |def rowRestriction: _root_.java.lang.String = ??? + """.stripMargin + case _ => "" + } + } +} + +final class BigQueryTypeInjector extends AnnotationTypeInjector { + import BigQueryTypeInjector._ + import AnnotationTypeInjector._ + + override def needsCompanionObject(source: ScTypeDefinition): Boolean = false + + override def injectFunctions(source: ScTypeDefinition): Seq[String] = + source match { + case c: ScClass if bqAnnotation(c).isDefined => + val parent = c.containingClass.getQualifiedName.init + val caseClasses = generatedCaseClasses(parent, c).find { c => + c.contains(CaseClassSuper) + } + getApplyPropsSignature(caseClasses).map(v => s"def $v = ???") + case _ => Seq.empty + } + + override def injectSupers(source: ScTypeDefinition): Seq[String] = + source match { + case c: ScClass if bqAnnotation(c).isDefined => Seq(CaseClassSuper) + case _ => Seq.empty + } + + /** + * Main method of the plugin. Injects syntactic inner members like case classes and companion + * objects, makes IntelliJ happy about BigQuery macros. Assumes macro is enclosed within + * class/object. + */ + override def injectInners(source: ScTypeDefinition): Seq[String] = { + source.extendsBlock.members + .collect { + case c: ScClass if bqAnnotation(c).isDefined => + val caseClasses = + generatedCaseClasses(source.getQualifiedName.init, c).find { c => + c.contains(CaseClassSuper) + } + (c, caseClasses) + } + .collect { + case (c, caseClasses) if caseClasses.nonEmpty => + val tupledMethod = getTupledMethod(c.getName, caseClasses) + val applyPropsSignature = + getApplyPropsSignature(caseClasses).mkString(",") + val unapplyReturnTypes = + getUnapplyReturnTypes(caseClasses).mkString(",") + + val extraCompanionMethod = + fetchExtraBQTypeCompanionMethods(source, c) + + // TODO: missing extends and traits - are they needed? + // $tn extends ${p(c, SType)}.HasSchema[$name] with ..$traits + s"""|object ${c.getName} { + | def apply( $applyPropsSignature ): ${c.getName} = ??? + | def unapply(x$$0: ${c.getName}): _root_.scala.Option[($unapplyReturnTypes)] = ??? + | def fromTableRow: _root_.scala.Function1[_root_.com.google.api.services.bigquery.model.TableRow, ${c.getName} ] = ??? + | def toTableRow: _root_.scala.Function1[ ${c.getName}, _root_.com.google.api.services.bigquery.model.TableRow] = ??? + | def schema: _root_.com.google.api.services.bigquery.model.TableSchema = ??? + | def toPrettyString(indent: Int = 0): String = ??? + | $extraCompanionMethod + | $tupledMethod + |}""".stripMargin + } + } +} diff --git a/src/main/scala/com/spotify/scio/ScioInjector.scala b/src/main/scala/com/spotify/scio/ScioInjector.scala deleted file mode 100644 index cb7aed0..0000000 --- a/src/main/scala/com/spotify/scio/ScioInjector.scala +++ /dev/null @@ -1,328 +0,0 @@ -/* - * Copyright 2016 Spotify AB. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.spotify.scio - -import java.io.File -import java.nio.charset.Charset -import java.nio.file.{Path, Paths, Files => JFiles} - -import com.google.common.base.Charsets -import com.google.common.hash.Hashing -import com.google.common.io.Files -import com.intellij.openapi.diagnostic.Logger -import com.intellij.psi.PsiElement -import org.jetbrains.plugins.scala.lang.psi.api.toplevel.typedef.{ - ScClass, - ScTypeDefinition -} -import org.jetbrains.plugins.scala.lang.psi.impl.toplevel.typedef.SyntheticMembersInjector - -import scala.collection.mutable -import com.intellij.notification.Notifications -import com.intellij.notification.Notification -import com.intellij.notification.NotificationType - -object ScioInjector { - private val Log = Logger.getInstance(classOf[ScioInjector]) - - // Could not find a way to get fully qualified annotation names - // even tho there is API, it does not return the annotations. - // For now stick with relative annotation names. - private val BQTNamespace = "BigQueryType" - private val FromQuery = s"$BQTNamespace.fromQuery" - private val FromTable = s"$BQTNamespace.fromTable" - private val FromStorage = s"$BQTNamespace.fromStorage" - private val Annotations = Seq( - FromQuery, - FromTable, - FromStorage, - s"$BQTNamespace.fromSchema", - s"$BQTNamespace.toTable" - ) - - private val AvroTNamespace = "AvroType" - private val AvroAnnotations = Seq( - s"$AvroTNamespace.fromSchema", - s"$AvroTNamespace.fromPath", - s"$AvroTNamespace.toSchema" - ) - - private val AlertEveryMissedXInvocations = 5 - - /** - * Finds BigQuery cache file, must be in sync with Scio implementation, otherwise plugin will - * not be able to find scala files. - */ - private def getClassCacheFile(filename: String): Path = { - val sysPropOverride = - Seq("generated.class.cache.directory", "bigquery.class.cache.directory") - .flatMap(sys.props.get) - .headOption - .map(Paths.get(_).resolve(filename)) - - sysPropOverride.getOrElse { - val oldBqPath = Paths - .get(sys.props("java.io.tmpdir")) - .resolve("bigquery-classes") - .resolve(filename) - val newBqPath = Paths - .get(sys.props("java.io.tmpdir")) - .resolve(sys.props("user.name")) - .resolve("bigquery-classes") - .resolve(filename) - val path = Paths - .get(sys.props("java.io.tmpdir")) - .resolve(sys.props("user.name")) - .resolve("generated-classes") - .resolve(filename) - - Seq(path, newBqPath, oldBqPath).find(JFiles.exists(_)).getOrElse(path) - } - } - - /** - * Computes hash for macro - the hash must be consistent with hash implementation in Scio. - */ - private def genHashForMacro(owner: String, srcFile: String): String = - Hashing - .murmur3_32() - .newHasher() - .putString(owner, Charsets.UTF_8) - .putString(srcFile, Charsets.UTF_8) - .hash() - .toString - - private def getApplyPropsSignature(caseClasses: Seq[String]) = - getConstructorProps(caseClasses) - .map(_.props) - .getOrElse(Seq.empty) - .mkString(" , ") - - private def fetchExtraBQTypeCompanionMethods( - source: ScTypeDefinition, - c: ScClass - ): String = { - val annotation = - c.annotations.map(_.getText).find(t => Annotations.exists(t.contains)).get - Log.debug(s"Found $annotation in ${source.getQualifiedNameForDebugger}") - - annotation match { - case a if a.contains(FromQuery) => - "def query: _root_.java.lang.String = ???" - case a if a.contains(FromTable) => - "def table: _root_.java.lang.String = ???" - case a if a.contains(FromStorage) => - """ - |def table: _root_.java.lang.String = ??? - |def selectedFields: _root_.scala.List[_root_.java.lang.String] = ??? - |def rowRestriction: _root_.java.lang.String = ??? - """.stripMargin - case _ => "" - } - } - - private def getConstructorProps( - caseClasses: Seq[String] - ): Option[ConstructorProps] = { - // TODO: duh. who needs regex ... but seriously tho, should this be regex? - caseClasses - .find( - c => - c.contains( - "extends _root_.com.spotify.scio.bigquery.types.BigQueryType.HasAnnotation" - ) || - c.contains( - "extends _root_.com.spotify.scio.avro.types.AvroType.HasAvroAnnotation" - ) - ) - .map( - _.split("[()]") - .filter(_.contains(" : ")) // get only parameter part - .flatMap(propsStr => { - val propsSplit = propsStr.split(",") - // We need to fix the split since Map types contain ',' as a part of their type declaration - val props = mutable.ArrayStack[String]() - for (prop <- propsSplit) { - if (prop.contains(" : ")) { - props += prop - } else { - assume(props.nonEmpty) - props += props.pop() + "," + prop - } - } - props.result.toList - }) - ) - .map(ConstructorProps(_)) // get individual parameter - } - - private[scio] def getUnapplyReturnTypes( - caseClasses: Seq[String] - ): Seq[String] = { - getConstructorProps(caseClasses).map(_.types).getOrElse(Seq.empty) - } - - private[scio] def getTupledMethod( - returnClassName: String, - caseClasses: Seq[String] - ): String = { - val maybeTupledMethod = getConstructorProps(caseClasses).map { - case cp: ConstructorProps if (2 to 22).contains(cp.types.size) => - s"def tupled: _root_.scala.Function1[( ${cp.types.mkString(" , ")} ), $returnClassName ] = ???" - case _ => - "" - } - - maybeTupledMethod.getOrElse("") - } - - final case class ConstructorProps(props: Seq[String]) { - val types: Seq[String] = props.map(_.split(" : ")(1).trim) - } -} - -final class ScioInjector extends SyntheticMembersInjector { - import ScioInjector._ - - private[this] val classMissed = - mutable.HashMap.empty[String, Int].withDefaultValue(0) - - override def needsCompanionObject(source: ScTypeDefinition): Boolean = false - - /** - * Main method of the plugin. Injects syntactic inner members like case classes and companion - * objects, makes IntelliJ happy about BigQuery macros. Assumes macro is enclosed within - * class/object. - */ - override def injectInners(source: ScTypeDefinition): Seq[String] = { - source.extendsBlock.members.flatMap { - case c: ScClass - if c.annotations - .map(_.getText) - .exists(t => Annotations.exists(t.contains)) => - val caseClasses = fetchGeneratedCaseClasses(source, c) - val extraCompanionMethod = fetchExtraBQTypeCompanionMethods(source, c) - val tupledMethod = getTupledMethod(c.getName, caseClasses) - - val applyPropsSignature = getApplyPropsSignature(caseClasses) - val unapplyReturnTypes = - getUnapplyReturnTypes(caseClasses).mkString(" , ") - - // TODO: missing extends and traits - are they needed? - // $tn extends ${p(c, SType)}.HasSchema[$name] with ..$traits - val companion = s"""|object ${c.getName} { - | def apply( $applyPropsSignature ): ${c.getName} = ??? - | def unapply(x$$0: ${c.getName}): _root_.scala.Option[($unapplyReturnTypes)] = ??? - | def fromTableRow: _root_.scala.Function1[_root_.com.google.api.services.bigquery.model.TableRow, ${c.getName} ] = ??? - | def toTableRow: _root_.scala.Function1[ ${c.getName}, _root_.com.google.api.services.bigquery.model.TableRow] = ??? - | def schema: _root_.com.google.api.services.bigquery.model.TableSchema = ??? - | def toPrettyString(indent: Int = 0): String = ??? - | $extraCompanionMethod - | $tupledMethod - |}""".stripMargin - - if (caseClasses.isEmpty) { - Seq.empty - } else { - companion +: caseClasses - } - - case c: ScClass - if c.annotations - .map(_.getText) - .exists(t => AvroAnnotations.exists(t.contains)) => - val caseClasses = fetchGeneratedCaseClasses(source, c) - val tupledMethod = getTupledMethod(c.getName, caseClasses) - val applyPropsSignature = getApplyPropsSignature(caseClasses) - val unapplyReturnTypes = - getUnapplyReturnTypes(caseClasses).mkString(" , ") - - val companion = s"""|object ${c.getName} { - | def apply( $applyPropsSignature ): ${c.getName} = ??? - | def unapply(x$$0: ${c.getName}): _root_.scala.Option[($unapplyReturnTypes)] = ??? - | def fromGenericRecord: _root_.scala.Function1[_root_.org.apache.avro.generic.GenericRecord, ${c.getName} ] = ??? - | def toGenericRecord: _root_.scala.Function1[ ${c.getName}, _root_.org.apache.avro.generic.GenericRecord] = ??? - | def schema: _root_.org.apache.avro.Schema = ??? - | def toPrettyString(indent: Int = 0): String = ??? - | $tupledMethod - |}""".stripMargin - - if (caseClasses.isEmpty) { - Seq.empty - } else { - companion +: caseClasses - } - case _ => Seq.empty - } - } - - private def fetchGeneratedCaseClasses( - source: ScTypeDefinition, - c: ScClass - ) = { - // For some reason sometimes [[getVirtualFile]] returns null, use Option. I don't know why. - val fileName = - Option(c.asInstanceOf[PsiElement].getContainingFile.getVirtualFile) - // wrap VirtualFile to java.io.File to use OS file separator - .map(vf => new File(vf.getCanonicalPath).getCanonicalPath) - - val hash = - fileName.map(genHashForMacro(source.getQualifiedNameForDebugger, _)) - - hash - .flatMap { h => - findClassFile(s"${c.getName}-$h.scala") - } - .map { f => - import collection.JavaConverters._ - Files - .readLines(f, Charset.defaultCharset()) - .asScala - .filter(_.contains("case class")) - } - .getOrElse(Seq.empty) - } - - private def findClassFile(fileName: String): Option[java.io.File] = { - val classFile = getClassCacheFile(fileName).toFile - val classFilePath = classFile.getAbsolutePath - if (classFile.exists()) { - Log.debug(s"Found $classFilePath") - classMissed(fileName) = 0 - Some(classFile) - } else { - classMissed(fileName) += 1 - val errorMessage = - s"""|Scio plugin could not find scala files for code completion. Please (re)compile the project. - |Missing: $classFilePath""".stripMargin - if (classMissed(fileName) >= AlertEveryMissedXInvocations) { - // reset counter - classMissed(fileName) = 0 - val notification = new Notification( - "ScioIDEA", - "Scio Plugin", - errorMessage, - NotificationType.ERROR - ) - Notifications.Bus.notify(notification) - } - Log.warn(errorMessage) - None - } - } -} diff --git a/src/test/scala/com/spotify/scio/ScioInjectorTest.scala b/src/test/scala/com/spotify/scio/ScioInjectorTest.scala index 44f3926..07e933f 100644 --- a/src/test/scala/com/spotify/scio/ScioInjectorTest.scala +++ b/src/test/scala/com/spotify/scio/ScioInjectorTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2017 Spotify AB. + * Copyright 2019 Spotify AB. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,85 +20,86 @@ package com.spotify.scio import org.scalatest.{FlatSpec, Matchers} class ScioInjectorTest extends FlatSpec with Matchers { - private val si = new ScioInjector() private val className = "Foobar" "Tupled method" should "be empty on case class with 1 parameter" in { - val input = Seq( + val input = Some( s"""|case class $className(f1 : _root_.scala.Option[_root_.java.lang.String]) - |extends _root_.com.spotify.scio.bigquery.types.BigQueryType.HasAnnotation - |""".stripMargin.replace("\n", " ") + |extends _root_.com.spotify.scio.bigquery.types.BigQueryType.HasAnnotation + |""".stripMargin.replace("\n", " ") ) - ScioInjector.getTupledMethod(className, input) shouldBe empty + AnnotationTypeInjector.getTupledMethod(className, input) shouldBe empty } it should "work on case class with 2 parameters" in { - val input = Seq( + val input = Some( s"""|case class $className(f1 : _root_.scala.Option[_root_.java.lang.String], - |f2 : _root_.scala.Option[_root_.java.lang.Long]) - |extends _root_.com.spotify.scio.bigquery.types.BigQueryType.HasAnnotation - |""".stripMargin.replace("\n", " ") + |f2 : _root_.scala.Option[_root_.java.lang.Long]) + |extends _root_.com.spotify.scio.bigquery.types.BigQueryType.HasAnnotation + |""".stripMargin.replace("\n", " ") ) val expected = s"def tupled: _root_.scala.Function1[( _root_.scala.Option[_root_.java.lang.String] , _root_.scala.Option[_root_.java.lang.Long] ), $className ] = ???" - ScioInjector.getTupledMethod(className, input) shouldBe expected + AnnotationTypeInjector.getTupledMethod(className, input) shouldBe expected } it should "be empty on case class more than 22 parameters" in { - val input = Seq( + val input = Some( s"""|case class $className(f1 : _root_.scala.Option[_root_.java.lang.String], - |f2 : _root_.scala.Option[_root_.java.lang.Long], - |f3 : _root_.scala.Option[_root_.java.lang.Long], - |f4 : _root_.scala.Option[_root_.java.lang.Long], - |f5 : _root_.scala.Option[_root_.java.lang.Long], - |f6 : _root_.scala.Option[_root_.java.lang.Long], - |f7 : _root_.scala.Option[_root_.java.lang.Long], - |f8 : _root_.scala.Option[_root_.java.lang.Long], - |f9 : _root_.scala.Option[_root_.java.lang.Long], - |f10 : _root_.scala.Option[_root_.java.lang.Long], - |f11 : _root_.scala.Option[_root_.java.lang.Long], - |f12 : _root_.scala.Option[_root_.java.lang.Long], - |f13 : _root_.scala.Option[_root_.java.lang.Long], - |f14 : _root_.scala.Option[_root_.java.lang.Long], - |f15 : _root_.scala.Option[_root_.java.lang.Long], - |f16 : _root_.scala.Option[_root_.java.lang.Long], - |f17 : _root_.scala.Option[_root_.java.lang.Long], - |f18 : _root_.scala.Option[_root_.java.lang.Long], - |f19 : _root_.scala.Option[_root_.java.lang.Long], - |f20 : _root_.scala.Option[_root_.java.lang.Long], - |f21 : _root_.scala.Option[_root_.java.lang.Long], - |f22 : _root_.scala.Option[_root_.java.lang.Long], - |f23 : _root_.scala.Option[_root_.java.lang.Long], - |extends _root_.com.spotify.scio.bigquery.types.BigQueryType.HasAnnotation - |""".stripMargin.replace("\n", " ") + |f2 : _root_.scala.Option[_root_.java.lang.Long], + |f3 : _root_.scala.Option[_root_.java.lang.Long], + |f4 : _root_.scala.Option[_root_.java.lang.Long], + |f5 : _root_.scala.Option[_root_.java.lang.Long], + |f6 : _root_.scala.Option[_root_.java.lang.Long], + |f7 : _root_.scala.Option[_root_.java.lang.Long], + |f8 : _root_.scala.Option[_root_.java.lang.Long], + |f9 : _root_.scala.Option[_root_.java.lang.Long], + |f10 : _root_.scala.Option[_root_.java.lang.Long], + |f11 : _root_.scala.Option[_root_.java.lang.Long], + |f12 : _root_.scala.Option[_root_.java.lang.Long], + |f13 : _root_.scala.Option[_root_.java.lang.Long], + |f14 : _root_.scala.Option[_root_.java.lang.Long], + |f15 : _root_.scala.Option[_root_.java.lang.Long], + |f16 : _root_.scala.Option[_root_.java.lang.Long], + |f17 : _root_.scala.Option[_root_.java.lang.Long], + |f18 : _root_.scala.Option[_root_.java.lang.Long], + |f19 : _root_.scala.Option[_root_.java.lang.Long], + |f20 : _root_.scala.Option[_root_.java.lang.Long], + |f21 : _root_.scala.Option[_root_.java.lang.Long], + |f22 : _root_.scala.Option[_root_.java.lang.Long], + |f23 : _root_.scala.Option[_root_.java.lang.Long], + |extends _root_.com.spotify.scio.bigquery.types.BigQueryType.HasAnnotation + |""".stripMargin.replace("\n", " ") ) - ScioInjector.getTupledMethod(className, input) shouldBe empty + AnnotationTypeInjector.getTupledMethod(className, input) shouldBe empty } it should "work on case class with Map field" in { - val input = Seq( + val input = Some( s"""|case class $className(f1 : _root_.scala.Option[_root_.java.lang.String], - |f2 : _root_.scala.Option[_root_.scala.collection.Map[_root_.java.lang.String, _root_.java.lang.String]]) - |extends _root_.com.spotify.scio.bigquery.types.BigQueryType.HasAnnotation - |""".stripMargin.replace("\n", " ") + |f2 : _root_.scala.Option[_root_.scala.collection.Map[_root_.java.lang.String, _root_.java.lang.String]]) + |extends _root_.com.spotify.scio.bigquery.types.BigQueryType.HasAnnotation + |""".stripMargin.replace("\n", " ") ) val expected = s"def tupled: _root_.scala.Function1[( _root_.scala.Option[_root_.java.lang.String] , _root_.scala.Option[_root_.scala.collection.Map[_root_.java.lang.String, _root_.java.lang.String]] ), $className ] = ???" - ScioInjector.getTupledMethod(className, input) shouldBe expected + AnnotationTypeInjector.getTupledMethod(className, input) shouldBe expected } it should "return the unapply return types on case class with 3 parameters" in { - val input = Seq(s"""|case class $className( - |f1 : _root_.scala.Option[_root_.java.lang.String], - |f2 : _root_.scala.Option[_root_.java.lang.Long], - |f2 : _root_.scala.Option[_root_.java.lang.Int]) - |extends _root_.com.spotify.scio.bigquery.types.BigQueryType.HasAnnotation - |""".stripMargin.replace("\n", " ")) + val input = Some( + s"""|case class $className( + |f1 : _root_.scala.Option[_root_.java.lang.String], + |f2 : _root_.scala.Option[_root_.java.lang.Long], + |f2 : _root_.scala.Option[_root_.java.lang.Int]) + |extends _root_.com.spotify.scio.bigquery.types.BigQueryType.HasAnnotation + |""".stripMargin.replace("\n", " ") + ) val expected = Seq( "_root_.scala.Option[_root_.java.lang.String]", "_root_.scala.Option[_root_.java.lang.Long]", "_root_.scala.Option[_root_.java.lang.Int]" ) - ScioInjector.getUnapplyReturnTypes(input) shouldBe expected + AnnotationTypeInjector.getUnapplyReturnTypes(input) shouldBe expected } } diff --git a/version.sbt b/version.sbt index 815b96d..bed27e3 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -ThisBuild / version := "0.1.18" +ThisBuild / version := "0.1.19"