4.0 - Integrating neo4j apoc

.gitignore

@@ -39,4 +39,3 @@ hadoop
 progress.csv
 logs/
 store_lock
-import/

CONTRIBUTING.adoc

@@ -53,6 +53,23 @@ If there are multiple commits, and the PR is fine to merge online, use *Squash a
 
 Using the issue id in the summary line will allow us to keep track of commits belonging together.
 
+=== Branching Policy
+
+We do have a branch for each major version of Neo4j, e.g. `3.5`, `4.0` and `4.1`.
+As a contributor you're asked to use newest possible branch for you PR.
+
+When your PR is accepted and merged it's the responsibility of the maintainers who merged it to cherry-pick that changes to any newer branch.
+Once cherry-picking is done, the maintainers should mark that PR with the `cherry-picked` label.
+
+Please indicate in your PR message text if your PR needs a different behaviour - e.g.  if the feature you're fixing has been removed in a newer branch or APIs have changed too much and you have a separate PR for the newer branch.
+
+EXAMPLE:
+
+1. You're fixing a bug being reported for 3.5.x.x.
+2. You're choosing the 3.5 branch as base for your PR branch.
+3. Once you're done you send a PR.
+4. When a maintainer merges that PR, they also take care to cherry-pick it to 4.0 and any more recent branches.
+
 === Handling pull requests
 
 Be polite. 
@@ -84,4 +101,4 @@ curl $PULL_REQUEST_URL.patch | git am --ignore-whitespace
 Make sure to push changes to a PR to the original remote branch.
 This will cause the pull request UI in GitHub show and link those commits.
 
-This guideline document is based on the https://github.com/spring-projects/spring-data-build/blob/master/CONTRIBUTING.adoc[spring-data guidelines], thanks @olivergierke.
+This guideline document is based on the https://github.com/spring-projects/spring-data-build/blob/master/CONTRIBUTING.adoc[spring-data guidelines], thanks @olivergierke.

build.gradle

@@ -1,20 +1,25 @@
+import org.gradle.api.internal.artifacts.DefaultExcludeRule
+
 plugins {
     id 'java'
-    id 'com.github.johnrengelman.shadow' version '4.0.2'
+    id 'com.github.johnrengelman.shadow' version '4.0.3'
     id "com.bmuschko.nexus" version "2.3.1"
 //    id "me.champeau.gradle.jmh" version "0.4.8"
     id 'maven-publish'
     id 'antlr'
+    id "org.sonarqube" version "2.7"
+    id "org.jetbrains.kotlin.jvm" version "1.3.71"
 }
 
+
+
 java {
     sourceCompatibility = JavaVersion.VERSION_11
     targetCompatibility = JavaVersion.VERSION_11
 }
 
 group = 'org.neo4j.procedure'
-//version = '4.0.0.1-SNAPSHOT'
-version = '4.0.0.0'
+version = '4.0.0.11'
 archivesBaseName = 'apoc'
 description = """neo4j-apoc-procedures"""
 
@@ -26,7 +31,7 @@ jar {
 
 ext {
     // NB: due to version.json generation by parsing this file, the next line must not have any if/then/else logic
-    neo4jVersion = "4.0.0"
+    neo4jVersion = "4.0.3"
     // instead we apply the override logic here
     neo4jVersionEffective = project.hasProperty("neo4jVersionOverride") ? project.getProperty("neo4jVersionOverride") : neo4jVersion
     testContainersVersion = '1.12.2'
@@ -50,7 +55,7 @@ configurations {
 }
 
 shadowJar {
-    classifier = 'all'  
+    archiveClassifier.set('all')
 }
 
 compileJava {
@@ -68,20 +73,25 @@ compileJava {
 //            "-ADocumentation.ExportGrouping=SINGLE,PACKAGE"]
 }
 
+compileKotlin {
+    kotlinOptions.jvmTarget = "1.8"
+}
+
 generateGrammarSource {
     arguments += ["-package", "apoc.custom"]
 }
 
 dependencies {
-
+    compileOnly "org.jetbrains.kotlin:kotlin-stdlib"
+    testCompile "org.jetbrains.kotlin:kotlin-stdlib"
 //    apt 'net.biville.florent:neo4j-sproc-compiler:1.2'  // temporarily disabled until byte[] is supported by sproc compiler
     apt group: 'org.neo4j', name: 'neo4j', version: neo4jVersionEffective	
-    compile group: 'commons-codec', name: 'commons-codec', version: '1.9'
+    compile group: 'commons-codec', name: 'commons-codec', version: '1.14'
     compileOnly group: 'com.sun.mail', name: 'javax.mail', version: '1.6.0'
     testCompile group: 'com.sun.mail', name: 'javax.mail', version: '1.6.0'
     compile group: 'com.jayway.jsonpath', name: 'json-path', version: '2.4.0'
     compile group: 'org.hdrhistogram', name: 'HdrHistogram', version: '2.1.9'
-    compile group: 'org.neo4j.driver', name: 'neo4j-java-driver', version: '4.0.0-beta03'
+    compile group: 'org.neo4j.driver', name: 'neo4j-java-driver', version: '4.0.1'
     compile group: 'com.novell.ldap', name: 'jldap', version: '2009-10-07'
 
     antlr "org.antlr:antlr4:4.7.2", {
@@ -148,8 +158,19 @@ dependencies {
 
     compileOnly group: 'org.neo4j', name: 'neo4j', version: neo4jVersionEffective
     compileOnly group: 'org.codehaus.jackson', name: 'jackson-mapper-asl', version: '1.9.7'
+
+    compileOnly group: 'com.fasterxml.jackson.module', name: 'jackson-module-kotlin', version: '2.10.3'
+    compileOnly 'org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.3.71'
+
+    testCompile group: 'com.fasterxml.jackson.module', name: 'jackson-module-kotlin', version: '2.10.3'
+    testCompile 'org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.3.71'
+
+
+    compileOnly group: 'com.amazonaws', name: 'aws-java-sdk-comprehend', version: '1.11.683'
+    testImplementation group: 'com.amazonaws', name: 'aws-java-sdk-comprehend', version: '1.11.683'
+
     testCompile group: 'org.codehaus.jackson', name: 'jackson-mapper-asl', version: '1.9.7'
-    compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.9.7'
+    compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.10.3'
     compile group: 'com.opencsv', name: 'opencsv', version: '4.2' 
     compileOnly group: 'org.ow2.asm', name: 'asm', version: '5.0.2'
     compile group: 'com.github.javafaker', name: 'javafaker', version: '0.10'
@@ -203,7 +224,7 @@ test {
     //exclude '**/CypherProceduresClusterTest.class'//, '**/AtomicTest.class'
 
     // neo4jDockerImage system property is used in TestContainerUtil
-    systemProperties 'user.language' : 'en' , 'user.country ' : 'US', 'neo4jDockerImage' : 'neo4j:4.0.0-rc01-enterprise'
+    systemProperties 'user.language' : 'en' , 'user.country ' : 'US', 'neo4jDockerImage' : "neo4j:${neo4jVersion}-enterprise"
 
 /*
     testLogging {
@@ -370,45 +391,69 @@ task myJavadocJar(type: Jar) {
 
 
 publishing {
-
-
     repositories {
         maven {
             name = 'pipeline'
             url = "file://${rootProject.buildDir}/repo"
         }
     }
     publications {
-        full(MavenPublication) {
-            pom {
-                name = "neo4j-apoc-procedure"
-                description = "A collection of useful Neo4j Procedures"
-                groupId = groupId
-                artifactId = archivesBaseName
-                version = version
-                scm { url = "http://github.com/neo4j-contrib/neo4j-apoc-procedures" }
-                url = 'http://github.com/neo4j-contrib/neo4j-apoc-procedures'
-
-                licenses {
-                    license {
-                        name ='The Apache Software License, Version 2.0'
-                        url ='http://www.apache.org/licenses/LICENSE-2.0.txt'
-                        comments ="""Note that this license is for the project itself, and not for its dependencies."""
-                        distribution= "repo"
+        shadow(MavenPublication) { publication ->
+            project.shadow.component(publication)
+            artifact(mySourcesJar)
+            artifact(myJavadocJar)
+            artifact(jar)
+
+            pom.withXml {
+                def root = asNode()
+                root.appendNode("name", "neo4j-apoc-procedure")
+                root.appendNode("description", "A collection of useful Neo4j Procedures")
+                root.appendNode("url", "http://github.com/neo4j-contrib/neo4j-apoc-procedures")
+
+                def scmNode = root.appendNode("scm")
+                scmNode.appendNode("url", "http://github.com/neo4j-contrib/neo4j-apoc-procedures")
+
+                def licensesNode = root.appendNode("licenses")
+                def licenseNode = licensesNode.appendNode("license")
+                licenseNode.appendNode("name", 'The Apache Software License, Version 2.0')
+                licenseNode.appendNode("url", 'http://www.apache.org/licenses/LICENSE-2.0.txt')
+                licenseNode.appendNode("comments", 'Note that this license is for the project itself, and not for its dependencies.')
+                licenseNode.appendNode("distribution", 'repo')
+
+                def developersNode = root.appendNode("developers")
+                def developerNode = developersNode.appendNode("developer")
+                developerNode.appendNode("id", 'michaelhunger')
+                developerNode.appendNode("name", 'Michael Hunger')
+                developerNode.appendNode("email", '[email protected]')
+
+                def dependenciesNode = root.get("dependencies")[0]
+
+                configurations.default.allDependencies.forEach {
+                    def dependency = dependenciesNode.appendNode('dependency')
+                    dependency.appendNode('groupId', it.group)
+                    dependency.appendNode('artifactId', it.name)
+                    dependency.appendNode('version', it.version)
+                    dependency.appendNode('scope', "compile")
+
+                    def myExcludeRules = it.excludeRules
+                    if (myExcludeRules.size() == 0) {
+                        myExcludeRules = new HashSet<>()
                     }
-                }
 
-                developers {
-                    developer {
-                        id ='michaelhunger'
-                        name= 'Michael Hunger'
-                        email= '[email protected]'
+                    myExcludeRules.add(new DefaultExcludeRule("org.slf4j", "slf4j-nop"))
+
+                    // for exclusions
+                    if (myExcludeRules.size() > 0) {
+                        def exclusions = dependency.appendNode('exclusions')
+                        myExcludeRules.each { ExcludeRule ex ->
+                            def exclusion = exclusions.appendNode('exclusion')
+                            exclusion.appendNode('artifactId', ex.module ? ex.module : "*")
+                            exclusion.appendNode('groupId', ex.group)
+                        }
                     }
+
                 }
 
-                from(components.java)
-                artifact(mySourcesJar)
-                artifact(myJavadocJar)
             }
 
         }

docs/asciidoc/_algorithms.adoc

@@ -10,9 +10,6 @@ include::algorithms/deprecated.adoc[]
 
 * <<path-finding-procedures>>
 * <<similarity>>
-* <<community-detection>>
-* <<centrality>>
-* <<pagerank>>
 
 include::algorithms/path-finding-procedures.adoc[leveloffset=2]
 include::algorithms/similarity.adoc[leveloffset=2]

docs/asciidoc/_nlp.adoc

@@ -0,0 +1,17 @@
+[[nlp]]
+== Natural Language Processing
+
+[abstract]
+--
+This chapter describes procedures that can be used for added Natural Language Processing functionality to graph applications.
+--
+
+The procedures described in this chapter act as wrappers around cloud based Natural Language APIs.
+
+This section includes:
+
+* <<nlp-gcp>>
+* <<nlp-aws>>
+
+include::nlp/gcp.adoc[]
+include::nlp/aws.adoc[]

docs/asciidoc/_path_finding.adoc

@@ -6,18 +6,25 @@
 This chapter describes advanced graph querying procedures in the APOC library.
 --
 
+The procedures in this chapter enable the querying of graph data when pure Cypher isn't enough.
 For more information on how to use these procedures, see:
 
 * <<path-expander>>
-* <<expand-subgraph>>
-* <<expand-spanning-tree>>
+** <<path-expander-paths>>
+** <<path-expander-paths-config>>
+** <<expand-subgraph-nodes>>
+** <<expand-subgraph>>
+** <<expand-spanning-tree>>
 * <<neighbourhood-search>>
 * <<path-functions>>
 * <<rel-functions>>
 * <<node-functions>>
 * <<parallel-node-search>>
 
+include::path-finding/expand-overview.adoc[leveloffset=1]
 include::path-finding/expand.adoc[leveloffset=1]
+include::path-finding/expand-config.adoc[leveloffset=1]
+include::path-finding/expand-subgraph-nodes.adoc[leveloffset=1]
 include::path-finding/expand-subgraph.adoc[leveloffset=1]
 include::path-finding/expand-spanning-tree.adoc[leveloffset=1]
 include::path-finding/neighborhood-search.adoc[leveloffset=1]

docs/asciidoc/algorithms/algo.adoc

@@ -1,29 +1,11 @@
 [[community-detection]]
-== Deprecated: Community Detection via Label Propagation
+== Removed: Community Detection via Label Propagation
 
 [abstract]
 --
 This section describes the Label Propagation community detection algorithm.
 --
 
-include::deprecated.adoc[]
+This algorithm has been deprecated by the https://neo4j.com/docs/graph-data-science/current/[Graph Data Science Library^], which has more performant and battle hardened versions of the algorithms.
 
-APOC includes a simple procedure for label propagation.
-It may be used to detect communities or solve other graph partitioning problems.
-The following example shows how it may be used.
-
-The example call with scan all nodes 25 times.
-During a scan the procedure will look at all outgoing relationships of type :X for each node n.
-For each of these relationships, it will compute a weight and use that as a vote for the other node's 'partition' property value.
-Finally, n.partition is set to the property value that acquired the most votes.
-
-Weights are computed by multiplying the relationship weight with the weight of the other nodes.
-Both weights are taken from the 'weight' property; if no such property is found, the weight is assumed to be 1.0.
-Similarly, if no 'weight' property key was specified, all weights are assumed to be 1.0.
-
-[source,cypher]
-----
-CALL apoc.algo.community(25,null,'partition','X','OUTGOING','weight',10000)
-----
-
-The second argument is a list of label names and may be used to restrict which nodes are scanned.
+For documentation of the Label Propagation algorithm, see https://neo4j.com/docs/graph-data-science/current/algorithms/label-propagation/