Restore titan-hbase support in hadoop2 branch

CHANGELOG.textile

@@ -5,17 +5,58 @@ Faunus: Graph Analytics Engine
 
 h2. Faunus 0.x.y
 
-h3. Version 0.4.1 (NOT OFFICIALLY RELEASED YET)
+h3. Version 0.4.4 (Apr 22, 2014)
 
 ```xml
 <dependency>
   <groupId>com.thinkaurelius.faunus</groupId>
   <artifactId>faunus</artifactId>
-  <version>0.4.1-SNAPSHOT</version>
+  <version>0.4.4</version>
+</dependency>
+```
+
+Incremented Titan version
+
+h3. Version 0.4.3 (Apr 21, 2014)
+
+```xml
+<dependency>
+  <groupId>com.thinkaurelius.faunus</groupId>
+  <artifactId>faunus</artifactId>
+  <version>0.4.3</version>
+</dependency>
+```
+
+* Added error handling for invalid RDF parameters
+* Added support for incremental edge loading with @BlueprintsGraphOutputMapReduce@
+* Bumped to support Titan 0.4.3
+
+h3. Version 0.4.2 (Jan 9, 2014)
+
+```xml
+<dependency>
+  <groupId>com.thinkaurelius.faunus</groupId>
+  <artifactId>faunus</artifactId>
+  <version>0.4.2</version>
+</dependency>
+```
+
+* Added Titan ElasticSearch dependency so its available in the Hadoop job jar
+* Bumped to support Titan 0.4.2
+
+h3. Version 0.4.1 (Nov 24, 2013)
+
+```xml
+<dependency>
+  <groupId>com.thinkaurelius.faunus</groupId>
+  <artifactId>faunus</artifactId>
+  <version>0.4.1</version>
 </dependency>
 ```
 
 * Fixed a severe bug in @Configuration@ entry orderings and @MapSequence@
+* Changed default Cassandra partitioner from Random to Murmur3
+* Broke @BlueprintsGraphOutputMapReduce@ into a MapReduce and then a Map (speeds up edge loading)
 
 h3. Version 0.4.0 (Oct 16, 2013)
 
@@ -40,7 +81,7 @@ h3. Version 0.4.0 (Oct 16, 2013)
 * Added support for @has(key)@ and @hasNot(key)@
 * Migrated from @Query.Compare@ to @Compare@ with Blueprints 2.4.0
 * The variables @hdfs@ and @local@ are available to @gremlin.sh -e@
-* Remove @SequenceFile@ migration model via Faunus (unsustainable)
+* Removed @SequenceFile@ migration model via Faunus (unsustainable)
 
 ==<hr/>==
 

README.textile

@@ -10,7 +10,7 @@ h2. Features
   *** "Apache HBase":http://hbase.apache.org/
   ** "Rexster":http://rexster.tinkerpop.com fronted graph databases
   ** "GraphSON":https://github.com/tinkerpop/blueprints/wiki/GraphSON-Reader-and-Writer-Library text format stored in HDFS
-  **  EdgeList multi-relational text format stored in HDFS
+  ** EdgeList multi-relational text format stored in HDFS
   *** "RDF":http://www.w3.org/RDF/ text formats stored in HDFS
   ** Hadoop binary "sequence files":http://wiki.apache.org/hadoop/SequenceFile stored in HDFS
   ** User defined import/export "scripts":https://github.com/thinkaurelius/faunus/wiki/Script-Format

bin/faunus.properties

@@ -23,3 +23,6 @@ faunus.output.location.overwrite=true
 # mapred.reduce.tasks=3
 # mapred.job.reuse.jvm.num.tasks=-1
 # mapred.task.timeout=5400000
+# mapred.reduce.parallel.copies=50
+# io.sort.factor=100
+# io.sort.mb=200

bin/titan-cassandra-output.properties

@@ -17,6 +17,7 @@ faunus.graph.output.titan.infer-schema=true
 # faunus.graph.output.blueprints.script-file=BlueprintsScript.groovy
 # controls size of transaction
 mapred.max.split.size=5242880
+# mapred.reduce.tasks=10
 mapred.job.reuse.jvm.num.tasks=-1
 
 faunus.sideeffect.output.format=org.apache.hadoop.mapreduce.lib.output.TextOutputFormat

bin/titan-hbase-output.properties

@@ -15,6 +15,7 @@ faunus.graph.output.titan.infer-schema=true
 # faunus.graph.output.blueprints.script-file=BlueprintsScript.groovy
 # controls size of transaction
 mapred.max.split.size=5242880
+# mapred.reduce.tasks=10
 mapred.job.reuse.jvm.num.tasks=-1
 
 faunus.sideeffect.output.format=org.apache.hadoop.mapreduce.lib.output.TextOutputFormat

data/BlueprintsScript.groovy

@@ -1,18 +1,22 @@
+import com.thinkaurelius.faunus.FaunusEdge
 import com.thinkaurelius.faunus.FaunusVertex
+import com.tinkerpop.blueprints.Edge
 import com.tinkerpop.blueprints.Graph
 import com.tinkerpop.blueprints.Vertex
+import com.tinkerpop.gremlin.java.GremlinPipeline
 import org.apache.hadoop.mapreduce.Mapper
 
 import static com.thinkaurelius.faunus.formats.BlueprintsGraphOutputMapReduce.Counters.*
 import static com.thinkaurelius.faunus.formats.BlueprintsGraphOutputMapReduce.LOGGER
 
 /**
- * This script is used to determine vertex uniqueness within a pre-existing graph.
- * If the vertex already exists in the graph, return it.
- * Else, if the vertex does not already exist, create it and return it.
- * Any arbitrary function can be implemented, but the one here implements an index lookup on a unique key.
+ * This script is used to determine vertex and edge uniqueness within a pre-existing graph.
+ * If the vertex/edge already exists in the graph, return it.
+ * Else, if the vertex/edge does not already exist, create it and return it.
+ * Any arbitrary function can be implemented. The two examples provided are typical scenarios.
  *
  * @author Marko A. Rodriguez (http://markorodriguez.com)
+ * @author Daniel Kuppitz (daniel at thinkaurelius.com)
  */
 def Vertex getOrCreateVertex(final FaunusVertex faunusVertex, final Graph graph, final Mapper.Context context) {
     final String uniqueKey = "name";
@@ -33,9 +37,33 @@ def Vertex getOrCreateVertex(final FaunusVertex faunusVertex, final Graph graph,
         context.getCounter(VERTICES_WRITTEN).increment(1l);
     }
 
+    // if vertex existed or not, add all the properties of the faunusVertex to the blueprintsVertex
     for (final String property : faunusVertex.getPropertyKeys()) {
         blueprintsVertex.setProperty(property, faunusVertex.getProperty(property));
         context.getCounter(VERTEX_PROPERTIES_WRITTEN).increment(1l);
     }
     return blueprintsVertex;
+}
+
+def Edge getOrCreateEdge(final FaunusEdge faunusEdge, final Vertex blueprintsOutVertex, final Vertex blueprintsInVertex, final Graph graph, final Mapper.Context context) {
+    final String edgeLabel = faunusEdge.getLabel();
+    final GremlinPipeline blueprintsEdgePipe = blueprintsOutVertex.outE(edgeLabel).as("e").inV().retain([blueprintsInVertex]).range(0, 1).back("e")
+    final Edge blueprintsEdge;
+
+    if (blueprintsEdgePipe.hasNext()) {
+        blueprintsEdge = blueprintsEdgePipe.next();
+        if (blueprintsEdgePipe.hasNext()) {
+            LOGGER.error("There's more than one edge labeled '" + edgeLabel + "' between vertex #" + blueprintsOutVertex.getId() + " and vertex #" + blueprintsInVertex.getId());
+        }
+    } else {
+        blueprintsEdge = graph.addEdge(null, blueprintsOutVertex, blueprintsInVertex, edgeLabel);
+        context.getCounter(EDGES_WRITTEN).increment(1l);
+    }
+
+    // if edge existed or not, add all the properties of the faunusEdge to the blueprintsEdge
+    for (final String key : faunusEdge.getPropertyKeys()) {
+        blueprintsEdge.setProperty(key, faunusEdge.getProperty(key));
+        context.getCounter(EDGE_PROPERTIES_WRITTEN).increment(1l);
+    }
+    return blueprintsEdge;
 }

pom.xml

@@ -9,7 +9,7 @@
     </parent>
     <groupId>com.thinkaurelius.faunus</groupId>
     <artifactId>faunus</artifactId>
-    <version>0.4.1-SNAPSHOT</version>
+    <version>0.4.4-hadoop2</version>
     <packaging>jar</packaging>
     <url>http://thinkaurelius.github.com/faunus/</url>
     <name>Faunus: Graph Analytics Engine</name>
@@ -19,8 +19,9 @@
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <sesame.version>2.6.10</sesame.version>
         <tinkerpop.version>2.4.0</tinkerpop.version>
-        <titan.version>0.4.1-SNAPSHOT</titan.version>
+        <titan.version>0.4.4-hadoop2</titan.version>
         <slf4j.version>1.7.5</slf4j.version>
+        <hadoop.version>2.2.0</hadoop.version>
     </properties>
     <developers>
         <developer>
@@ -52,8 +53,8 @@
         <connection>scm:git:[email protected]:thinkaurelius/faunus.git</connection>
         <developerConnection>scm:git:[email protected]:thinkaurelius/faunus.git</developerConnection>
         <url>[email protected]:thinkaurelius/faunus.git</url>
-        <tag>HEAD</tag>
-    </scm>
+      <tag>0.4.4</tag>
+  </scm>
     <dependencyManagement>
         <dependencies>
             <dependency>
@@ -92,7 +93,7 @@
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-client</artifactId>
-            <version>2.2.0</version>
+            <version>${hadoop.version}</version>
         </dependency>
         <!-- TITAN -->
         <dependency>
@@ -105,11 +106,16 @@
             <artifactId>titan-cassandra</artifactId>
             <version>${titan.version}</version>
         </dependency>
-        <!--<dependency>
+        <dependency>
             <groupId>com.thinkaurelius.titan</groupId>
             <artifactId>titan-hbase</artifactId>
             <version>${titan.version}</version>
-        </dependency>-->
+        </dependency>
+        <dependency>
+            <groupId>com.thinkaurelius.titan</groupId>
+            <artifactId>titan-es</artifactId>
+            <version>${titan.version}</version>
+        </dependency>
         <!-- RDF PARSING -->
         <dependency>
             <groupId>org.openrdf.sesame</groupId>
@@ -162,13 +168,6 @@
             <scope>test</scope>
         </dependency>
     </dependencies>
-    <repositories>
-        <repository>
-            <id>aduna-repo</id>
-            <name>Aduna repository</name>
-            <url>http://repo.aduna-software.org/maven2/releases</url>
-        </repository>
-    </repositories>
     <build>
         <!-- Used during release:perform to upload to S3 -->
         <extensions>
@@ -302,7 +301,14 @@
             </plugin>
             <plugin>
                 <artifactId>maven-release-plugin</artifactId>
-                <version>2.4.1</version>
+                <version>2.5</version>
+                <dependencies>
+                    <dependency>
+                        <groupId>org.apache.maven.scm</groupId>
+                        <artifactId>maven-scm-provider-gitexe</artifactId>
+                        <version>1.9</version>
+                    </dependency>
+                </dependencies>
                 <configuration>
                     <goals>deploy</goals>
                     <pushChanges>false</pushChanges>
@@ -366,6 +372,33 @@
                         </executions>
                     </plugin>
 
+                    <!--
+                       The distribution descriptor includes files written to the build
+                       directory by both the standalone and hadoop-job descriptors.
+                       That's why the execution for the distribution descriptor comes last.
+                    -->
+                    <plugin>
+                        <artifactId>maven-assembly-plugin</artifactId>
+                        <version>2.4</version>
+                        <executions>
+                            <execution>
+                                <id>assemble-zip-and-tar</id>
+                                <phase>package</phase>
+                                <goals>
+                                    <goal>single</goal>
+                                </goals>
+                                <configuration>
+                                    <attach>false</attach>
+                                    <finalName>${project.artifactId}</finalName>
+                                    <tarLongFileMode>gnu</tarLongFileMode>
+                                    <descriptors>
+                                        <descriptor>src/assembly/distribution.xml</descriptor>
+                                    </descriptors>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+
                     <plugin>
                         <groupId>org.codehaus.mojo</groupId>
                         <artifactId>exec-maven-plugin</artifactId>
@@ -434,33 +467,43 @@
                                     <workingDirectory>${project.basedir}</workingDirectory>
                                 </configuration>
                             </execution>
-                        </executions>
-                    </plugin>
-
-                    <!--
-                       The distribution descriptor includes files written to the build
-                       directory by both the standalone and hadoop-job descriptors.
-                       That's why the execution for the distribution descriptor comes last.
-                    -->
-                    <plugin>
-                        <artifactId>maven-assembly-plugin</artifactId>
-                        <version>2.4</version>
-                        <executions>
+                            <!-- GPG plugin can't sign a file.  It can only either:
+                                 * sign all artifacts attached to the project or
+                                 * sign and deploy a file
+                            -->
                             <execution>
-                                <id>build-attached-assemblies</id>
+                                <id>sign-zip</id>
                                 <phase>package</phase>
                                 <goals>
-                                    <goal>single</goal>
+                                    <goal>exec</goal>
                                 </goals>
                                 <configuration>
-                                    <attach>true</attach>
-                                    <finalName>${project.artifactId}</finalName>
-                                    <tarLongFileMode>gnu</tarLongFileMode>
-                                    <descriptors>
-                                        <descriptor>src/assembly/distribution.xml</descriptor>
-                                    </descriptors>
+                                    <executable>gpg</executable>
+                                    <arguments>
+                                        <argument>--detach-sign</argument>
+                                        <argument>--armor</argument>
+                                        <argument>${project.artifactId}-${project.version}.zip</argument>
+                                    </arguments>
+                                    <workingDirectory>${project.build.directory}</workingDirectory>
                                 </configuration>
                             </execution>
+                            <execution>
+                                <id>sign-tar</id>
+                                <phase>package</phase>
+                                <goals>
+                                    <goal>exec</goal>
+                                </goals>
+                                <configuration>
+                                    <executable>gpg</executable>
+                                    <arguments>
+                                        <argument>--detach-sign</argument>
+                                        <argument>--armor</argument>
+                                        <argument>${project.artifactId}-${project.version}.tar.bz2</argument>
+                                    </arguments>
+                                    <workingDirectory>${project.build.directory}</workingDirectory>
+                                </configuration>
+                            </execution>
+
                         </executions>
                     </plugin>
 

src/main/java/com/thinkaurelius/faunus/FaunusElement.java

@@ -30,7 +30,7 @@ public abstract class FaunusElement implements Element, WritableComparable<Faunu
         WritableComparator.define(FaunusElement.class, new Comparator());
     }
 
-    protected static final KryoSerializer serialize = new KryoSerializer(true);
+    protected static final KryoSerializer serialize = new KryoSerializer();
 
     protected static final Map<String, String> TYPE_MAP = new HashMap<String, String>() {
         @Override