This is an automated email from the ASF dual-hosted git repository. Cole-Greer pushed a commit to branch docsBuild-3.8 in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit 47114fa4a7a6894e23de0cf4d6508b4ceb3b1f7a Author: Cole Greer <[email protected]> AuthorDate: Sat May 2 22:20:13 2026 -0700 Replace AWK/shell docs preprocessing with AsciidoctorJ extension Introduce the gremlin-docs module, a custom AsciidoctorJ TreeProcessor extension that replaces the entire AWK/shell preprocessing pipeline (10 AWK scripts, 5 shell scripts) used to build TinkerPop documentation. The new system executes [gremlin-groovy] code blocks in an embedded GremlinGroovyScriptEngine during Asciidoctor rendering, eliminating the need for a running Gremlin Server, Hadoop daemons, or the Gremlin Console distribution at build time. Key features: - Embedded execution of gremlin code blocks with live query results - Auto-generated language variant tabs (Java, Python, JavaScript, C#, Go) using the ANTLR-based GremlinTranslator infrastructure - Standalone tab group support for manually-authored [source,LANG,tab] blocks - Hadoop/Spark example support via local-mode Spark with sandboxed HDFS - Console utility functions (describeGraph) loaded from gremlin-console - GremlinPlugin SPI loading for hadoop-gremlin/spark-gremlin imports and bindings - Syntax highlighting via highlight.js 11.9.0 (replaces CodeRay) - Console command detection (:remote, :>, :submit) for static rendering - Multi-line statement joining with bracket balancing - Callout marker preservation through execution New files: - gremlin-docs/ - Maven module (not in reactor, built separately) - GremlinDocsExtension.java - SPI entry point for AsciidoctorJ - GremlinTreeProcessor.java - Main TreeProcessor that walks the AST - GremlinExecutor.java - Embedded script engine wrapper - VariantTranslator.java - GremlinTranslator wrapper for all GLVs - GremlinExecutorTest.java - Unit tests - bin/process-docs-new.sh - New build entry point Root pom.xml changes: - Added gremlin-docs as asciidoctor-maven-plugin dependency - Switched source-highlighter from coderay to highlightjs 11.9.0 - Added highlightjs-languages for groovy support - Added tabs-1 CSS rule for single-tab blocks Usage: bin/process-docs-new.sh # full build bin/process-docs-new.sh --dry-run # skip gremlin execution Assisted-by: Kiro:claude-opus-4.6 --- bin/process-docs-new.sh | 82 ++++ docs/stylesheets/tinkerpop.css | 2 +- gremlin-docs/pom.xml | 98 +++++ .../gremlin/docs/GremlinDocsExtension.java | 34 ++ .../tinkerpop/gremlin/docs/GremlinExecutor.java | 445 +++++++++++++++++++++ .../gremlin/docs/GremlinTreeProcessor.java | 380 ++++++++++++++++++ .../tinkerpop/gremlin/docs/VariantTranslator.java | 132 ++++++ ...ciidoctor.jruby.extension.spi.ExtensionRegistry | 1 + .../gremlin/docs/GremlinExecutorTest.java | 141 +++++++ pom.xml | 69 +++- 10 files changed, 1368 insertions(+), 16 deletions(-) diff --git a/bin/process-docs-new.sh b/bin/process-docs-new.sh new file mode 100755 index 0000000000..576accf67a --- /dev/null +++ b/bin/process-docs-new.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Builds TinkerPop documentation using the gremlin-docs AsciidoctorJ extension. +# This bypasses the old AWK preprocessing pipeline and processes [gremlin-*] blocks +# directly during Asciidoctor rendering. +# +# Usage: +# bin/process-docs-new.sh # full build with live gremlin execution +# bin/process-docs-new.sh --dry-run # skip gremlin execution (fast, for layout checks) + +set -e + +PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "${PROJECT_ROOT}" + +TP_VERSION=$(cat pom.xml | grep -A1 '<artifactId>tinkerpop</artifactId>' | grep '<version>' | sed -e 's/.*<version>//' -e 's/<\/version>.*//') + +if [ -z "${TP_VERSION}" ]; then + echo "ERROR: Could not determine TinkerPop version from pom.xml" + exit 1 +fi + +ASCIIDOC_ATTRS="" +if [ "$1" = "--dry-run" ]; then + ASCIIDOC_ATTRS="-Dasciidoctor.attributes.gremlin-docs-dryrun=true" + echo "Dry-run mode: gremlin blocks will not be executed" +fi + +echo "Building docs for TinkerPop ${TP_VERSION}..." +echo "Source: docs/src/" +echo "Output: target/docs/htmlsingle/" + +# build and install the gremlin-docs extension (not part of the main reactor) +echo "Installing gremlin-docs extension..." +mvn install -f gremlin-docs/pom.xml -DskipTests -Denforcer.skip=true -q + +# copy static assets that live outside docs/src/ into the staging area +# (Maven's copy-docs-to-work-area handles docs/src/ itself) +mkdir -p target/doc-source +cp -r docs/static target/doc-source/ 2>/dev/null || true +cp -r docs/stylesheets target/doc-source/ 2>/dev/null || true + +# set up conf/hadoop so GraphFactory.open('conf/hadoop/...') resolves during build +mkdir -p conf/hadoop +cp hadoop-gremlin/conf/* conf/hadoop/ 2>/dev/null || true + +# run asciidoctor with the gremlin-docs extension, pointing at raw sources +mvn process-resources \ + -Dasciidoc \ + -Dasciidoc.source.dir="${PROJECT_ROOT}/docs/src" \ + -Drat.skip=true \ + ${ASCIIDOC_ATTRS} + +# clean up +rm -rf conf/hadoop +rmdir conf 2>/dev/null || true + +# post-process: replace version placeholder +echo "Post-processing: replacing x.y.z with ${TP_VERSION}..." +find target/docs/htmlsingle -name '*.html' | while IFS= read -r f; do + sed "s/x\.y\.z/${TP_VERSION}/g" "$f" > "$f.tmp" && mv "$f.tmp" "$f" +done + +echo "Done. Output in target/docs/htmlsingle/" diff --git a/docs/stylesheets/tinkerpop.css b/docs/stylesheets/tinkerpop.css index 71cc47eeec..b763310c03 100644 --- a/docs/stylesheets/tinkerpop.css +++ b/docs/stylesheets/tinkerpop.css @@ -692,4 +692,4 @@ table.tableblock.grid-all th.tableblock, table.tableblock.grid-all td.tableblock #footer { background-color: #465158; padding: 2em; } #footer-text { color: #eee; font-size: 0.8em; text-align: center; } -.tabs{position:relative;margin:40px auto;width:1024px;max-width:100%;overflow:hidden;padding-top:10px;margin-bottom:60px}.tabs input{position:absolute;z-index:1000;height:50px;left:0;top:0;opacity:0;-ms-filter:"progid:DXImageTransform.Microsoft.Alpha(Opacity=0)";filter:alpha(opacity=0);cursor:pointer;margin:0}.tabs input:hover+label{background:#e08f24}.tabs label{background:#e9ffe9;color:#1a1a1a;font-size:15px;line-height:50px;height:60px;position:relative;top:0;padding:0 20px;float:left [...] +.tabs{position:relative;margin:40px auto;width:1024px;max-width:100%;overflow:hidden;padding-top:10px;margin-bottom:60px}.tabs input{position:absolute;z-index:1000;height:50px;left:0;top:0;opacity:0;-ms-filter:"progid:DXImageTransform.Microsoft.Alpha(Opacity=0)";filter:alpha(opacity=0);cursor:pointer;margin:0}.tabs input:hover+label{background:#e08f24}.tabs label{background:#e9ffe9;color:#1a1a1a;font-size:15px;line-height:50px;height:60px;position:relative;top:0;padding:0 20px;float:left [...] diff --git a/gremlin-docs/pom.xml b/gremlin-docs/pom.xml new file mode 100644 index 0000000000..6840e9e964 --- /dev/null +++ b/gremlin-docs/pom.xml @@ -0,0 +1,98 @@ +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.tinkerpop</groupId> + <artifactId>tinkerpop</artifactId> + <version>3.8.2-SNAPSHOT</version> + </parent> + <artifactId>gremlin-docs</artifactId> + <name>Apache TinkerPop :: Gremlin Docs</name> + <description>AsciidoctorJ extension for processing Gremlin code blocks in TinkerPop documentation</description> + <dependencies> + <dependency> + <groupId>org.apache.tinkerpop</groupId> + <artifactId>gremlin-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.tinkerpop</groupId> + <artifactId>gremlin-groovy</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.tinkerpop</groupId> + <artifactId>tinkergraph-gremlin</artifactId> + <version>${project.version}</version> + </dependency> + <!-- optional: for [gremlin-groovy,GRAPH,hadoop] blocks that need OLAP/Spark --> + <dependency> + <groupId>org.apache.tinkerpop</groupId> + <artifactId>hadoop-gremlin</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.tinkerpop</groupId> + <artifactId>spark-gremlin</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.tinkerpop</groupId> + <artifactId>gremlin-console</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.asciidoctor</groupId> + <artifactId>asciidoctorj</artifactId> + <version>2.5.8</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.asciidoctor</groupId> + <artifactId>asciidoctorj-api</artifactId> + <version>2.5.8</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + <!-- test --> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.hamcrest</groupId> + <artifactId>hamcrest</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>ch.qos.logback</groupId> + <artifactId>logback-classic</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <directory>${basedir}/target</directory> + <finalName>${project.artifactId}-${project.version}</finalName> + </build> +</project> diff --git a/gremlin-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinDocsExtension.java b/gremlin-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinDocsExtension.java new file mode 100644 index 0000000000..5261b07700 --- /dev/null +++ b/gremlin-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinDocsExtension.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import org.asciidoctor.Asciidoctor; +import org.asciidoctor.jruby.extension.spi.ExtensionRegistry; + +/** + * SPI entry point that registers the {@link GremlinTreeProcessor} with AsciidoctorJ. + * Discovered automatically via {@code META-INF/services/org.asciidoctor.jruby.extension.spi.ExtensionRegistry}. + */ +public class GremlinDocsExtension implements ExtensionRegistry { + + @Override + public void register(final Asciidoctor asciidoctor) { + asciidoctor.javaExtensionRegistry().treeprocessor(GremlinTreeProcessor.class); + } +} diff --git a/gremlin-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinExecutor.java b/gremlin-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinExecutor.java new file mode 100644 index 0000000000..9bd228f857 --- /dev/null +++ b/gremlin-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinExecutor.java @@ -0,0 +1,445 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import org.apache.tinkerpop.gremlin.groovy.jsr223.GremlinGroovyScriptEngine; +import org.apache.tinkerpop.gremlin.jsr223.BindingsCustomizer; +import org.apache.tinkerpop.gremlin.jsr223.Customizer; +import org.apache.tinkerpop.gremlin.jsr223.GremlinPlugin; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.script.Bindings; +import javax.script.ScriptException; +import java.io.Closeable; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.ServiceLoader; +import java.util.stream.Stream; + +/** + * Wraps a {@link GremlinGroovyScriptEngine} to execute Gremlin code blocks and capture console-style output. + * Maintains state across evaluations within a single document, matching the behavior of the old AWK pipeline + * which piped the entire document through one Gremlin Console session. + */ +public class GremlinExecutor implements Closeable { + + private static final Logger log = LoggerFactory.getLogger(GremlinExecutor.class); + + /** + * Maximum number of results to display per traversal, matching the Gremlin Console's + * {@code :set max-iteration 100} default used by the old docs preprocessor. + * Can be changed per-block via {@code :set max-iteration N}. + */ + private int maxIteration = 100; + + private final GremlinGroovyScriptEngine engine; + private boolean hadoopInitialized; + + public GremlinExecutor() { + // Load all GremlinPlugin customizers (hadoop, spark, etc.) so their imports + // and bindings (hdfs, fs, spark, SparkGraphComputer, etc.) are available + final List<Customizer> customizers = new ArrayList<>(); + final List<BindingsCustomizer> bindingsCustomizers = new ArrayList<>(); + for (final GremlinPlugin plugin : ServiceLoader.load(GremlinPlugin.class)) { + plugin.getCustomizers("gremlin-groovy").ifPresent(c -> { + for (final Customizer customizer : c) { + if (customizer instanceof BindingsCustomizer) { + bindingsCustomizers.add((BindingsCustomizer) customizer); + } else { + customizers.add(customizer); + } + } + }); + } + this.engine = new GremlinGroovyScriptEngine(customizers.toArray(new Customizer[0])); + + // Set Hadoop's default filesystem to an isolated temp directory so that + // hdfs.ls(), hdfs.copyFromLocal() etc. operate in a clean sandbox instead + // of the user's home directory. + java.io.File hadoopTmp = null; + try { + hadoopTmp = java.nio.file.Files.createTempDirectory("tinkerpop-docs-hdfs").toFile(); + hadoopTmp.deleteOnExit(); + } catch (final Exception e) { + log.debug("Could not set up isolated HDFS directory", e); + } + + // BindingsCustomizer is not handled by the engine constructor — apply manually. + for (final BindingsCustomizer bc : bindingsCustomizers) { + final Bindings bindings = bc.getBindings(); + bindings.forEach((k, v) -> engine.put(k, v)); + } + + // Override hdfs/fs bindings with FileSystemStorage rooted at the temp directory. + // FileSystemStorage.ls() with no args lists fs.getHomeDirectory(), so we need a + // filesystem whose home directory is our temp dir. + if (hadoopTmp != null) { + try { + final String tmpPath = hadoopTmp.getAbsolutePath(); + engine.put("__docsHdfsRoot", tmpPath); + // Use a RawLocalFileSystem subclass that overrides getHomeDirectory. + // We define it in Groovy so it's available to the script engine. + engine.eval( + "class DocsLocalFileSystem extends org.apache.hadoop.fs.RawLocalFileSystem {\n" + + " private org.apache.hadoop.fs.Path home\n" + + " DocsLocalFileSystem(String homeDir) {\n" + + " super()\n" + + " this.home = new org.apache.hadoop.fs.Path(homeDir)\n" + + " initialize(java.net.URI.create('file:///'), new org.apache.hadoop.conf.Configuration())\n" + + " setWorkingDirectory(home)\n" + + " }\n" + + " org.apache.hadoop.fs.Path getHomeDirectory() { home }\n" + + "}\n" + + "hdfs = org.apache.tinkerpop.gremlin.hadoop.structure.io.FileSystemStorage.open(new DocsLocalFileSystem(__docsHdfsRoot))\n" + + "fs = hdfs\n"); + } catch (final Exception e) { + log.debug("Could not override hdfs binding", e); + } + } + + this.hadoopInitialized = false; + + // Load console utility functions (describeGraph, etc.) from gremlin-console + try (final java.io.InputStream is = Thread.currentThread().getContextClassLoader() + .getResourceAsStream("org/apache/tinkerpop/gremlin/console/jsr223/UtilitiesGremlinPluginScript.groovy")) { + if (is != null) { + engine.eval(new String(is.readAllBytes())); + } + } catch (final Exception e) { + log.debug("Could not load console utility functions", e); + } + } + + /** + * Initializes the graph environment for a code block. The graph parameter corresponds to the second + * attribute in {@code [gremlin-groovy,modern]} — e.g. "modern", "classic", "crew", "sink", "grateful", + * or empty for a bare TinkerGraph. "existing" means reuse the current graph state. + * <p> + * Replicates the initialization from the old {@code init-code-blocks.awk}: + * <ul> + * <li>Creates the graph via {@code TinkerFactory} or opens an empty {@code TinkerGraph}</li> + * <li>Creates a traversal source {@code g}</li> + * <li>Pre-binds {@code marko} vertex (if present in the graph) for convenience</li> + * <li>Cleans up {@code /tmp/tinkergraph.kryo} temp files</li> + * </ul> + */ + public void initGraph(final String graph) throws ScriptException { + initGraph(graph, false); + } + + /** + * Initializes the graph environment with optional Hadoop/Spark support. + * + * @param graph the graph name (modern, classic, etc.) or null for empty TinkerGraph + * @param hadoop if true, configures a HadoopGraph with Spark in local mode + */ + public void initGraph(final String graph, final boolean hadoop) throws ScriptException { + if ("existing".equals(graph)) return; + + if (hadoop) { + initHadoopGraph(graph); + return; + } + + // close previous graph if one exists + try { engine.eval("if (graph != null && graph instanceof AutoCloseable) graph.close()"); } + catch (final Exception ignored) { } + + if (graph != null && !graph.isEmpty()) { + engine.eval("graph = org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory.create" + + capitalize(graph) + "()"); + } else { + engine.eval("graph = org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph.open()"); + } + engine.eval("g = graph.traversal()"); + + // pre-bind convenience variables matching init-code-blocks.awk + engine.eval("marko = g.V().has('name', 'marko').tryNext().orElse(null)"); + engine.eval("f = new File('/tmp/tinkergraph.kryo'); if (f.exists()) f.deleteDir()"); + + } + + /** + * Initializes a HadoopGraph with Spark running in local mode. This enables execution of + * OLAP examples that use {@code SparkGraphComputer} without requiring external Hadoop/Spark + * infrastructure. The graph data is written to a temp file in Gryo format and read by + * HadoopGraph via the local filesystem. + */ + private void initHadoopGraph(final String graph) throws ScriptException { + // close previous graph if one exists + try { engine.eval("if (graph != null && graph instanceof AutoCloseable) graph.close()"); } + catch (final Exception ignored) { } + + if (!hadoopInitialized) { + // one-time setup: import hadoop/spark classes + engine.eval("import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph\n" + + "import org.apache.tinkerpop.gremlin.hadoop.Constants\n" + + "import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat\n" + + "import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat\n" + + "import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer\n" + + "import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoRegistrator\n" + + "import org.apache.tinkerpop.gremlin.hadoop.structure.io.FileSystemStorage\n" + + "import org.apache.commons.configuration2.BaseConfiguration\n" + + "import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo\n"); + hadoopInitialized = true; + } + + // write the TinkerFactory graph to a temp gryo file for HadoopGraph to read + final String factoryMethod = (graph != null && !graph.isEmpty()) + ? "TinkerFactory.create" + capitalize(graph) + "()" + : "TinkerGraph.open()"; + + engine.eval( + "tmpGraph = " + factoryMethod + "\n" + + "tmpFile = File.createTempFile('tinkerpop-docs-', '.kryo')\n" + + "tmpFile.deleteOnExit()\n" + + "tmpGraph.io(GryoIo.build()).writeGraph(tmpFile.absolutePath)\n" + + "tmpGraph.close()\n" + + "\n" + + "hadoopConf = new BaseConfiguration()\n" + + "hadoopConf.setProperty('gremlin.graph', 'org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph')\n" + + "hadoopConf.setProperty('gremlin.hadoop.graphReader', 'org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat')\n" + + "hadoopConf.setProperty('gremlin.hadoop.graphWriter', 'org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat')\n" + + "hadoopConf.setProperty('gremlin.hadoop.inputLocation', tmpFile.absolutePath)\n" + + "hadoopConf.setProperty('gremlin.hadoop.outputLocation', 'output-' + System.currentTimeMillis())\n" + + "hadoopConf.setProperty('gremlin.hadoop.jarsInDistributedCache', false)\n" + + "hadoopConf.setProperty('gremlin.hadoop.defaultGraphComputer', 'org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer')\n" + + "hadoopConf.setProperty('spark.master', 'local[4]')\n" + + "hadoopConf.setProperty('spark.serializer', 'org.apache.spark.serializer.KryoSerializer')\n" + + "hadoopConf.setProperty('spark.kryo.registrator', 'org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoRegistrator')\n" + + "\n" + + "graph = HadoopGraph.open(hadoopConf)\n" + + "g = traversal().with(graph).withComputer(SparkGraphComputer)\n"); + + } + + /** + * Executes a block of Gremlin code lines and returns console-style formatted output. + * Multi-line statements (lines ending with {@code .} for method chaining) are joined before + * evaluation. Results are formatted as {@code gremlin> line} followed by {@code ==>result} + * lines, matching the Gremlin Console output format. + * <p> + * When a block contains {@code import} statements, the entire block is evaluated as a single + * script since imports don't persist across separate {@code eval()} calls. + */ + public String execute(final List<String> lines) throws ScriptException { + // reset per-block settings + maxIteration = 100; + // check if block contains import statements — if so, evaluate as a single script + final boolean hasImports = lines.stream().anyMatch(l -> l.trim().startsWith("import ")); + if (hasImports) { + return executeAsScript(lines); + } + return executeLineByLine(lines); + } + + /** + * Evaluates the entire block as a single Groovy script. Used for blocks containing + * {@code import} statements or complex Groovy constructs that don't work with + * line-by-line evaluation. + */ + private String executeAsScript(final List<String> lines) throws ScriptException { + final StringBuilder output = new StringBuilder(); + final StringBuilder script = new StringBuilder(); + + for (final String rawLine : lines) { + final String trimmed = rawLine.replaceAll("(\\s*<\\d+>)+\\s*$", "").trim(); + if (trimmed.isEmpty()) continue; + if (trimmed.startsWith(":")) continue; + if (trimmed.startsWith("//")) continue; + + output.append("gremlin> ").append(trimmed).append("\n"); + script.append(trimmed).append("\n"); + } + + try { + final Object result = engine.eval(script.toString()); + if (result != null) { + formatResult(result, output); + } + } catch (final ScriptException e) { + log.warn("Error evaluating gremlin script block", e); + output.append("ERROR: ").append(e.getMessage()).append("\n"); + } + + return output.toString(); + } + + private String executeLineByLine(final List<String> lines) throws ScriptException { + final StringBuilder output = new StringBuilder(); + final StringBuilder currentStatement = new StringBuilder(); + + for (final String rawLine : lines) { + // strip trailing AsciiDoc callout markers like <1>, <2>, or multiple <5> <6> + final String trimmed = rawLine.replaceAll("(\\s*<\\d+>)+\\s*$", "").trim(); + if (trimmed.isEmpty()) continue; + + // handle :set max-iteration console command + if (trimmed.startsWith(":set max-iteration")) { + try { + maxIteration = Integer.parseInt(trimmed.split("\\s+")[2]); + } catch (final Exception ignored) { } + continue; + } + + // skip other console commands like :plugin, etc. + if (trimmed.startsWith(":")) continue; + + // skip comment lines + if (trimmed.startsWith("//")) continue; + + // accumulate multi-line statements (lines ending with . are continuations) + if (currentStatement.length() == 0) { + output.append("gremlin> ").append(trimmed).append("\n"); + } else { + output.append(" ").append(trimmed).append("\n"); + } + currentStatement.append(trimmed).append("\n"); + + // if line ends with a continuation character, keep accumulating + if (isContinuationLine(trimmed, currentStatement.toString())) { + continue; + } + + // evaluate the complete statement + final String stmt = currentStatement.toString(); + currentStatement.setLength(0); + + try { + final Object result = engine.eval(stmt); + if (result != null) { + formatResult(result, output); + } + } catch (final ScriptException e) { + log.warn("Error evaluating gremlin: {}", stmt, e); + output.append("ERROR: ").append(e.getMessage()).append("\n"); + } + } + + // evaluate any remaining accumulated statement + if (currentStatement.length() > 0) { + try { + final Object result = engine.eval(currentStatement.toString()); + if (result != null) { + formatResult(result, output); + } + } catch (final ScriptException e) { + log.warn("Error evaluating gremlin: {}", currentStatement, e); + } + } + + return output.toString(); + } + + /** + * Returns the raw Gremlin lines suitable for translation — strips comments, callout markers, + * and multi-line continuations into single statements. + */ + public static List<String> extractTranslatableLines(final List<String> lines) { + final List<String> result = new ArrayList<>(); + final StringBuilder current = new StringBuilder(); + + for (String line : lines) { + // strip trailing callout markers like <1>, <2>, or multiple <5> <6> + line = line.replaceAll("(\\s*<\\d+>)+\\s*$", "").trim(); + if (line.isEmpty() || line.startsWith("//") || line.startsWith(":")) continue; + + current.append(line).append("\n"); + + if (!isContinuationLine(line, current.toString())) { + result.add(current.toString().trim()); + current.setLength(0); + } + } + + if (current.length() > 0) { + result.add(current.toString().trim()); + } + + return result; + } + + /** + * Determines if the current accumulated statement is incomplete and needs more lines. + * Used by both {@link #executeLineByLine} and {@link #extractTranslatableLines}. + * <p> + * Note: counts brackets naively without respecting string literals. + * Sufficient for typical Gremlin doc examples. + */ + static boolean isContinuationLine(final String trimmedLine, final String accumulated) { + if (trimmedLine.endsWith(".") || trimmedLine.endsWith("{") || trimmedLine.endsWith(",") || + trimmedLine.endsWith("(") || trimmedLine.endsWith("\\")) { + return true; + } + return countChar(accumulated, '(') > countChar(accumulated, ')') || + countChar(accumulated, '[') > countChar(accumulated, ']') || + countChar(accumulated, '{') > countChar(accumulated, '}'); + } + + @Override + public void close() { + // GremlinGroovyScriptEngine does not implement Closeable/AutoCloseable. + // Clean up graph and Hadoop/Spark resources if they were initialized. + try { engine.eval("if (graph != null && graph instanceof AutoCloseable) graph.close()"); } + catch (final Exception ignored) { } + if (hadoopInitialized) { + try { + engine.eval("org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer.close()"); + } catch (final Exception ignored) { } + } + } + + private void formatResult(final Object result, final StringBuilder output) { + if (result instanceof Iterator) { + final Iterator<?> iter = (Iterator<?>) result; + int count = 0; + while (iter.hasNext() && count < maxIteration) { + output.append("==>").append(iter.next()).append("\n"); + count++; + } + } else if (result instanceof Iterable) { + int count = 0; + for (final Object item : (Iterable<?>) result) { + if (count >= maxIteration) break; + output.append("==>").append(item).append("\n"); + count++; + } + } else if (result instanceof Stream) { + ((Stream<?>) result).limit(maxIteration) + .forEach(item -> output.append("==>").append(item).append("\n")); + } else { + output.append("==>").append(result).append("\n"); + } + } + + private static String capitalize(final String s) { + if (s == null || s.isEmpty()) return s; + return Character.toUpperCase(s.charAt(0)) + s.substring(1); + } + + private static int countChar(final String s, final char c) { + int count = 0; + for (int i = 0; i < s.length(); i++) { + if (s.charAt(i) == c) count++; + } + return count; + } +} diff --git a/gremlin-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinTreeProcessor.java b/gremlin-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinTreeProcessor.java new file mode 100644 index 0000000000..5562b3bb0a --- /dev/null +++ b/gremlin-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinTreeProcessor.java @@ -0,0 +1,380 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import org.asciidoctor.ast.Block; +import org.asciidoctor.ast.Document; +import org.asciidoctor.ast.StructuralNode; +import org.asciidoctor.extension.Treeprocessor; +import org.apache.tinkerpop.gremlin.language.translator.Translator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * AsciidoctorJ {@link Treeprocessor} that processes {@code [gremlin-groovy,modern]} code blocks + * in TinkerPop documentation. For each such block, it: + * <ol> + * <li>Executes the Gremlin code in an embedded {@link GremlinExecutor} and captures console output</li> + * <li>Translates the canonical Gremlin to all language variants via {@link VariantTranslator}</li> + * <li>Wraps the console output and translations in a tabbed UI with proper AST listing blocks + * so Asciidoctor applies syntax highlighting via CodeRay</li> + * </ol> + */ +public class GremlinTreeProcessor extends Treeprocessor { + + private static final Logger log = LoggerFactory.getLogger(GremlinTreeProcessor.class); + private static final Pattern GREMLIN_STYLE = Pattern.compile("gremlin-(\\w+)"); + private static final AtomicLong counter = new AtomicLong(System.currentTimeMillis()); + + @Override + public Document process(final Document document) { + final boolean dryRun = document.hasAttribute("gremlin-docs-dryrun"); + + try (final GremlinExecutor executor = new GremlinExecutor()) { + processNode(document, executor, dryRun); + } + + return document; + } + + private void processNode(final StructuralNode node, final GremlinExecutor executor, final boolean dryRun) { + final List<StructuralNode> blocks = node.getBlocks(); + if (blocks == null || blocks.isEmpty()) return; + + for (int i = 0; i < blocks.size(); i++) { + final StructuralNode child = blocks.get(i); + + if (child instanceof Block && isGremlinBlock((Block) child)) { + i = processGremlinBlock(node, i, (Block) child, executor, dryRun); + } else if (child instanceof Block && isTabStartBlock((Block) child)) { + i = processStandaloneTabGroup(node, i); + } else { + processNode(child, executor, dryRun); + } + } + } + + /** + * Replaces a gremlin block with a sequence of AST nodes that form a tabbed view: + * passthrough HTML for the tab structure interleaved with real listing blocks that + * Asciidoctor will syntax-highlight. + */ + private int processGremlinBlock(final StructuralNode parent, final int index, + final Block block, final GremlinExecutor executor, + final boolean dryRun) { + final Matcher m = GREMLIN_STYLE.matcher(block.getStyle()); + if (!m.matches()) return index; + + final String lang = m.group(1); + final String graph = getGraphAttribute(block); + final boolean hadoop = isHadoopBlock(block); + final List<String> lines = block.getLines(); + + log.info("Processing [gremlin-{},{}{}] block ({} lines)", lang, + graph != null ? graph : "", hadoop ? ",hadoop" : "", lines.size()); + + // execute the gremlin code + String consoleOutput; + if (dryRun || isConsoleCommandBlock(lines)) { + consoleOutput = formatDryRun(lines); + } else { + try { + executor.initGraph(graph, hadoop); + consoleOutput = executor.execute(lines); + } catch (final Exception e) { + log.error("Failed to execute gremlin block", e); + consoleOutput = formatDryRun(lines); + } + } + + // collect tab entries: label + language + code content + final List<TabEntry> tabs = new ArrayList<>(); + tabs.add(new TabEntry("console", "groovy", consoleOutput)); + + // translate to language variants (available on 4.0+ with ANTLR-based translator) + final List<String> translatableLines = GremlinExecutor.extractTranslatableLines(lines); + if (!translatableLines.isEmpty()) { + final Map<Translator, String> translations = VariantTranslator.translateBlock(translatableLines); + for (final Map.Entry<Translator, String> entry : translations.entrySet()) { + tabs.add(new TabEntry( + VariantTranslator.getDisplayName(entry.getKey()), + VariantTranslator.getSourceLanguage(entry.getKey()), + entry.getValue())); + } + } + + // consume any following [source,LANG,tab] blocks + final List<StructuralNode> siblings = parent.getBlocks(); + int nextIndex = index + 1; + while (nextIndex < siblings.size()) { + final StructuralNode next = siblings.get(nextIndex); + if (next instanceof Block && isManualTabBlock((Block) next)) { + final Block tabBlock = (Block) next; + final String tabLang = getSourceLanguage(tabBlock); + tabs.add(new TabEntry( + tabLang != null ? tabLang : "code", + tabLang, + String.join("\n", tabBlock.getLines()))); + nextIndex++; + } else { + break; + } + } + + // build the replacement AST nodes + final List<StructuralNode> replacements = buildTabbedBlocks(parent, tabs); + + // replace original block and consumed tab blocks with the new sequence + // remove consumed blocks first (backwards to preserve indices) + for (int j = nextIndex - 1; j > index; j--) { + siblings.remove(j); + } + // remove the original gremlin block + siblings.remove(index); + // insert replacements at the same position + siblings.addAll(index, replacements); + + // return last index of inserted blocks so the loop continues after them + return index + replacements.size() - 1; + } + + /** + * Builds a sequence of AST blocks: passthrough HTML for tab structure interleaved + * with real listing blocks for syntax-highlighted code. + */ + private List<StructuralNode> buildTabbedBlocks(final StructuralNode parent, final List<TabEntry> tabs) { + final List<StructuralNode> nodes = new ArrayList<>(); + + final long id = counter.incrementAndGet(); + final int numTabs = tabs.size(); + + // opening HTML: section + radio buttons + labels + first tab content div open + final StringBuilder openHtml = new StringBuilder(); + openHtml.append("<section class=\"tabs tabs-").append(numTabs).append("\">\n"); + for (int i = 0; i < numTabs; i++) { + final int tabNum = i + 1; + final String checked = (i == 0) ? " checked=\"checked\"" : ""; + openHtml.append(" <input id=\"tab-").append(id).append("-").append(tabNum) + .append("\" type=\"radio\" name=\"radio-set-").append(id) + .append("\" class=\"tab-selector-").append(tabNum).append("\"") + .append(checked).append(" />\n"); + openHtml.append(" <label for=\"tab-").append(id).append("-").append(tabNum) + .append("\" class=\"tab-label-").append(tabNum).append("\">") + .append(tabs.get(i).label).append("</label>\n"); + } + openHtml.append(" <div class=\"tabcontent\">\n <div class=\"tabcontent-1\">\n"); + nodes.add(createBlock(parent, "pass", openHtml.toString())); + + // first tab content (listing block) + nodes.add(createListingBlock(parent, tabs.get(0).language, tabs.get(0).content)); + + // remaining tabs: close previous div, open next div, listing block + for (int i = 1; i < numTabs; i++) { + final int tabNum = i + 1; + final String divHtml = " </div>\n </div>\n" + + " <div class=\"tabcontent\">\n <div class=\"tabcontent-" + tabNum + "\">\n"; + nodes.add(createBlock(parent, "pass", divHtml)); + nodes.add(createListingBlock(parent, tabs.get(i).language, tabs.get(i).content)); + } + + // closing HTML + nodes.add(createBlock(parent, "pass", " </div>\n </div>\n</section>")); + + return nodes; + } + + /** + * Creates a proper Asciidoctor source listing block by parsing AsciiDoc markup. + * This ensures CodeRay syntax highlighting is applied, since the block goes through + * Asciidoctor's normal parsing pipeline. + */ + private Block createListingBlock(final StructuralNode parent, final String language, final String content) { + final List<String> lines = new ArrayList<>(); + lines.add("[source," + language + "]"); + lines.add("----"); + for (final String line : content.split("\n", -1)) { + lines.add(line); + } + lines.add("----"); + final int sizeBefore = parent.getBlocks().size(); + parseContent(parent, lines); + final List<StructuralNode> blocks = parent.getBlocks(); + if (blocks.size() > sizeBefore) { + return (Block) blocks.remove(blocks.size() - 1); + } + // fallback if parseContent produced nothing + return (Block) createBlock(parent, "listing", content); + } + + private boolean isGremlinBlock(final Block block) { + final String style = block.getStyle(); + return style != null && GREMLIN_STYLE.matcher(style).matches(); + } + + /** + * Checks if a block starts a standalone tab group: {@code [source,LANG,tab]}. + */ + private boolean isTabStartBlock(final Block block) { + if (!"source".equals(block.getStyle())) return false; + final Map<String, Object> attrs = block.getAttributes(); + // "tab" can appear as attribute "2" or "3" depending on how asciidoctor parses positions + return "tab".equals(attrs.get("2")) || "tab".equals(attrs.get("3")); + } + + /** + * Checks if a block is a continuation of a tab group: a {@code [source,LANG]} block + * whose language hasn't already been seen in the group. + */ + private boolean isTabContinuationBlock(final Block block, final java.util.Set<String> seenLanguages) { + if (!"source".equals(block.getStyle())) return false; + final String lang = getSourceLanguage(block); + return lang != null && !seenLanguages.contains(lang); + } + + private boolean isManualTabBlock(final Block block) { + if (!"source".equals(block.getStyle())) return false; + final Map<String, Object> attrs = block.getAttributes(); + return "tab".equals(attrs.get("2")) || "tab".equals(attrs.get("3")); + } + + /** + * Processes a standalone tab group starting with {@code [source,LANG,tab]} and collecting + * all consecutive {@code [source,LANG]} blocks into a tabbed view. + */ + private int processStandaloneTabGroup(final StructuralNode parent, final int index) { + final List<StructuralNode> siblings = parent.getBlocks(); + final List<TabEntry> tabs = new ArrayList<>(); + + // collect the first block and all consecutive source blocks + final Set<String> seenLanguages = new HashSet<>(); + int nextIndex = index; + while (nextIndex < siblings.size()) { + final StructuralNode node = siblings.get(nextIndex); + if (!(node instanceof Block)) break; + final Block block = (Block) node; + + if (nextIndex == index) { + // first block must be a tab-start block + if (!isTabStartBlock(block)) break; + } else { + // subsequent blocks must be source blocks with a unique language + if (!isTabContinuationBlock(block, seenLanguages)) break; + } + + final String lang = getSourceLanguage(block); + final String label = lang != null ? lang : "code"; + if (lang != null) seenLanguages.add(lang); + tabs.add(new TabEntry(label, lang, String.join("\n", block.getLines()))); + nextIndex++; + } + + if (tabs.size() <= 1) return index; // not enough blocks for tabs + + log.info("Processing standalone tab group ({} tabs)", tabs.size()); + + final List<StructuralNode> replacements = buildTabbedBlocks(parent, tabs); + + // remove original blocks (backwards) + for (int j = nextIndex - 1; j >= index; j--) { + siblings.remove(j); + } + siblings.addAll(index, replacements); + + return index + replacements.size() - 1; + } + + private String getGraphAttribute(final Block block) { + final Map<String, Object> attrs = block.getAttributes(); + Object attr = attrs.get("2"); + if (attr == null) attr = attrs.get(2); + if (attr == null || "false".equals(attr.toString()) || attr.toString().isEmpty()) return null; + return attr.toString(); + } + + /** + * Checks if a gremlin block has the "hadoop" attribute, e.g. {@code [gremlin-groovy,modern,hadoop]}. + * The "hadoop" flag appears as the third positional attribute. + */ + private boolean isHadoopBlock(final Block block) { + final Map<String, Object> attrs = block.getAttributes(); + Object attr = attrs.get("3"); + if (attr == null) attr = attrs.get(3); + return "hadoop".equals(attr != null ? attr.toString() : null); + } + + private String getSourceLanguage(final Block block) { + // For [source,LANG], asciidoctor may store the language in "language" attr, + // attribute "1" (which may contain "source"), or attribute "2". + final Object langAttr = block.getAttribute("language"); + if (langAttr != null) return langAttr.toString(); + final Map<String, Object> attrs = block.getAttributes(); + // attribute "1" is often the style name itself; "2" has the language + final Object attr2 = attrs.get("2"); + if (attr2 != null && !"tab".equals(attr2.toString()) && !"false".equals(attr2.toString())) { + return attr2.toString(); + } + final Object attr1 = attrs.get("1"); + if (attr1 != null && !"source".equals(attr1.toString())) return attr1.toString(); + return null; + } + + /** + * Detects blocks that contain console commands ({@code :remote}, {@code :>}, + * {@code :submit}) which cannot be executed in an embedded engine. These are rendered + * as static code blocks with {@code gremlin>} prompts. + */ + private static boolean isConsoleCommandBlock(final List<String> lines) { + for (final String line : lines) { + final String trimmed = line.trim(); + if (trimmed.startsWith(":remote") || trimmed.startsWith(":>") || trimmed.startsWith(":submit")) { + return true; + } + } + return false; + } + + private static String formatDryRun(final List<String> lines) { + final StringBuilder sb = new StringBuilder(); + for (final String line : lines) { + sb.append("gremlin> ").append(line).append("\n"); + } + return sb.toString(); + } + + private static class TabEntry { + final String label; + final String language; + final String content; + + TabEntry(final String label, final String language, final String content) { + this.label = label; + this.language = language; + this.content = content; + } + } +} diff --git a/gremlin-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/VariantTranslator.java b/gremlin-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/VariantTranslator.java new file mode 100644 index 0000000000..ef5bd8b64a --- /dev/null +++ b/gremlin-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/VariantTranslator.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import org.apache.tinkerpop.gremlin.language.translator.GremlinTranslator; +import org.apache.tinkerpop.gremlin.language.translator.Translation; +import org.apache.tinkerpop.gremlin.language.translator.Translator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Translates canonical Gremlin into all supported language variants using {@link GremlinTranslator}. + */ +public class VariantTranslator { + + private static final Logger log = LoggerFactory.getLogger(VariantTranslator.class); + + /** + * The language variants to generate, in display order. Excludes CANONICAL, ANONYMIZED, GROOVY + * (which is essentially the same as the console output), and LANGUAGE (deprecated). + */ + static final List<Translator> VARIANT_LANGUAGES = Collections.unmodifiableList(Arrays.asList( + Translator.JAVA, + Translator.PYTHON, + Translator.JAVASCRIPT, + Translator.DOTNET, + Translator.GO + )); + + /** + * Display names for tab labels. + */ + private static final Map<Translator, String> DISPLAY_NAMES = new LinkedHashMap<>(); + static { + DISPLAY_NAMES.put(Translator.JAVA, "java"); + DISPLAY_NAMES.put(Translator.PYTHON, "python"); + DISPLAY_NAMES.put(Translator.JAVASCRIPT, "javascript"); + DISPLAY_NAMES.put(Translator.DOTNET, "c#"); + DISPLAY_NAMES.put(Translator.GO, "go"); + } + + /** + * Asciidoc source language identifiers for syntax highlighting. + */ + private static final Map<Translator, String> SOURCE_LANGUAGES = new LinkedHashMap<>(); + static { + SOURCE_LANGUAGES.put(Translator.JAVA, "java"); + SOURCE_LANGUAGES.put(Translator.PYTHON, "python"); + SOURCE_LANGUAGES.put(Translator.JAVASCRIPT, "javascript"); + SOURCE_LANGUAGES.put(Translator.DOTNET, "csharp"); + SOURCE_LANGUAGES.put(Translator.GO, "go"); + } + + public static String getDisplayName(final Translator translator) { + return DISPLAY_NAMES.getOrDefault(translator, translator.getName().toLowerCase()); + } + + public static String getSourceLanguage(final Translator translator) { + return SOURCE_LANGUAGES.getOrDefault(translator, translator.getName().toLowerCase()); + } + + /** + * Translates a single Gremlin statement to all variant languages. Returns a map from + * {@link Translator} to the translated code string. Statements that fail to parse + * (e.g. those containing lambdas or non-standard Groovy) are skipped with a warning. + */ + public static Map<Translator, String> translateStatement(final String gremlin) { + final Map<Translator, String> results = new LinkedHashMap<>(); + for (final Translator lang : VARIANT_LANGUAGES) { + try { + final Translation t = GremlinTranslator.translate(gremlin, "g", lang); + results.put(lang, t.getTranslated()); + } catch (final Exception e) { + log.debug("Cannot translate to {}: {} — {}", lang.getName(), gremlin, e.getMessage()); + } + } + return results; + } + + /** + * Translates multiple Gremlin statements and joins them with newlines per language. + * If any statement fails to translate for a given language, that language is omitted entirely. + */ + public static Map<Translator, String> translateBlock(final List<String> statements) { + final Map<Translator, String> results = new LinkedHashMap<>(); + + for (final Translator lang : VARIANT_LANGUAGES) { + final StringBuilder sb = new StringBuilder(); + boolean allTranslated = true; + + for (final String stmt : statements) { + try { + final Translation t = GremlinTranslator.translate(stmt, "g", lang); + if (sb.length() > 0) sb.append("\n"); + sb.append(t.getTranslated()); + } catch (final Exception e) { + log.debug("Cannot translate to {}: {} — {}", lang.getName(), stmt, e.getMessage()); + allTranslated = false; + break; + } + } + + if (allTranslated) { + results.put(lang, sb.toString()); + } + } + + return results; + } +} diff --git a/gremlin-docs/src/main/resources/META-INF/services/org.asciidoctor.jruby.extension.spi.ExtensionRegistry b/gremlin-docs/src/main/resources/META-INF/services/org.asciidoctor.jruby.extension.spi.ExtensionRegistry new file mode 100644 index 0000000000..6a81ac1f60 --- /dev/null +++ b/gremlin-docs/src/main/resources/META-INF/services/org.asciidoctor.jruby.extension.spi.ExtensionRegistry @@ -0,0 +1 @@ +org.apache.tinkerpop.gremlin.docs.GremlinDocsExtension diff --git a/gremlin-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinExecutorTest.java b/gremlin-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinExecutorTest.java new file mode 100644 index 0000000000..c03f6d2d45 --- /dev/null +++ b/gremlin-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinExecutorTest.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import org.apache.tinkerpop.gremlin.language.translator.Translator; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.*; + +public class GremlinExecutorTest { + + @Test + public void shouldExecuteSimpleTraversal() throws Exception { + try (final GremlinExecutor executor = new GremlinExecutor()) { + executor.initGraph("modern"); + final String output = executor.execute(Arrays.asList("g.V().count()")); + assertTrue(output.contains("gremlin> g.V().count()")); + assertTrue(output.contains("==>6")); + } + } + + @Test + public void shouldMaintainStateBetweenExecutions() throws Exception { + try (final GremlinExecutor executor = new GremlinExecutor()) { + executor.initGraph("modern"); + executor.execute(Arrays.asList("x = g.V().has('name','marko').next()")); + + // "existing" should reuse the graph and bindings + executor.initGraph("existing"); + final String output = executor.execute(Arrays.asList("x.value('name')")); + assertTrue(output.contains("==>marko")); + } + } + + @Test + public void shouldExecuteMultipleLines() throws Exception { + try (final GremlinExecutor executor = new GremlinExecutor()) { + executor.initGraph("modern"); + final String output = executor.execute(Arrays.asList( + "g.V().has('name','marko').values('name')", + "g.V().has('name','marko').out('knows').values('name')" + )); + assertTrue(output.contains("==>marko")); + assertTrue(output.contains("==>josh")); + assertTrue(output.contains("==>vadas")); + } + } + + @Test + public void shouldExtractTranslatableLines() { + final List<String> lines = Arrays.asList( + "g.V().has('name','marko'). <1>", + " out('knows').values('name') <2>", + "// this is a comment", + "g.V().count()" + ); + final List<String> result = GremlinExecutor.extractTranslatableLines(lines); + assertEquals(2, result.size()); + assertEquals("g.V().has('name','marko').\nout('knows').values('name')", result.get(0)); + assertEquals("g.V().count()", result.get(1)); + } + + @Test + public void shouldTranslateToVariants() { + final Map<Translator, String> translations = VariantTranslator.translateStatement( + "g.V().has('name','marko').out('knows').values('name')"); + + assertFalse(translations.isEmpty()); + assertTrue(translations.containsKey(Translator.PYTHON)); + assertTrue(translations.containsKey(Translator.JAVA)); + assertTrue(translations.containsKey(Translator.JAVASCRIPT)); + assertTrue(translations.containsKey(Translator.DOTNET)); + assertTrue(translations.containsKey(Translator.GO)); + + // python should use snake_case + assertTrue(translations.get(Translator.PYTHON).contains("has(")); + assertTrue(translations.get(Translator.PYTHON).contains("out(")); + } + + @Test + public void shouldTranslateBlock() { + final List<String> statements = Arrays.asList( + "g.V().has('name','marko').out('knows').values('name')", + "g.V().count()" + ); + final Map<Translator, String> translations = VariantTranslator.translateBlock(statements); + + assertFalse(translations.isEmpty()); + // each translation should contain both statements + for (final String code : translations.values()) { + assertTrue(code.contains("\n")); + } + } + + @Test + public void shouldInitEmptyGraph() throws Exception { + try (final GremlinExecutor executor = new GremlinExecutor()) { + executor.initGraph(null); + final String output = executor.execute(Arrays.asList("g.V().count()")); + assertTrue(output.contains("==>0")); + } + } + + @Test + public void shouldInitEmptyStringGraph() throws Exception { + try (final GremlinExecutor executor = new GremlinExecutor()) { + executor.initGraph(""); + final String output = executor.execute(Arrays.asList("g.V().count()")); + assertTrue(output.contains("==>0")); + } + } + + @Test + public void shouldSkipUntranslatableStatements() { + // lambdas can't be translated + final Map<Translator, String> translations = VariantTranslator.translateStatement( + "g.V().filter{it.get().label() == 'person'}"); + // should either be empty or have partial results — not throw + assertNotNull(translations); + } +} diff --git a/pom.xml b/pom.xml index aaef1b2fde..03d4878173 100644 --- a/pom.xml +++ b/pom.xml @@ -956,9 +956,9 @@ limitations under the License. </activation> <properties> - <!-- the source asciidocs are copied from /docs to asciidoc.source.dir by bin/process-docs.sh - where they can the scripts embedded in the text are then executed and their results shoved - back into the doc --> + <!-- When using the gremlin-docs AsciidoctorJ extension, point directly at docs/src + by passing -Dasciidoc.source.dir=docs/src on the command line. The old preprocessor + pipeline writes to target/postprocess-asciidoc which remains the default. --> <asciidoc.source.dir>${project.basedir}/target/postprocess-asciidoc</asciidoc.source.dir> <!-- once code processing is over, the documents are basically ready for the formatter and are @@ -1035,6 +1035,21 @@ limitations under the License. <groupId>org.asciidoctor</groupId> <artifactId>asciidoctor-maven-plugin</artifactId> <inherited>false</inherited> + <dependencies> + <dependency> + <groupId>org.apache.tinkerpop</groupId> + <artifactId>gremlin-docs</artifactId> + <version>${project.version}</version> + </dependency> + <!-- Force commons-text version to match what gremlin-core needs. + The asciidoctor plugin bundles commons-text:1.3 which conflicts + with commons-configuration2's requirement for 1.15.0. --> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-text</artifactId> + <version>1.15.0</version> + </dependency> + </dependencies> <executions> <execution> <id>home</id> @@ -1065,7 +1080,9 @@ limitations under the License. <encoding>UTF-8</encoding> <stylesdir>${asciidoctor.style.dir}</stylesdir> <stylesheet>tinkerpop.css</stylesheet> - <source-highlighter>coderay</source-highlighter> + <source-highlighter>highlightjs</source-highlighter> + <highlightjsdir>https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0</highlightjsdir> + <highlightjs-languages>groovy</highlightjs-languages> <basedir>${project.basedir}</basedir> <docinfo>shared</docinfo> <docinfodir>${project.basedir}/docs/src</docinfodir> @@ -1092,7 +1109,9 @@ limitations under the License. <toc-position>left</toc-position> <stylesdir>${asciidoctor.style.dir}</stylesdir> <stylesheet>tinkerpop.css</stylesheet> - <source-highlighter>coderay</source-highlighter> + <source-highlighter>highlightjs</source-highlighter> + <highlightjsdir>https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0</highlightjsdir> + <highlightjs-languages>groovy</highlightjs-languages> <basedir>${project.basedir}</basedir> <docinfo>shared</docinfo> <docinfodir>${project.basedir}/docs/src</docinfodir> @@ -1119,7 +1138,9 @@ limitations under the License. <toc-position>left</toc-position> <stylesdir>${asciidoctor.style.dir}</stylesdir> <stylesheet>tinkerpop.css</stylesheet> - <source-highlighter>coderay</source-highlighter> + <source-highlighter>highlightjs</source-highlighter> + <highlightjsdir>https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0</highlightjsdir> + <highlightjs-languages>groovy</highlightjs-languages> <basedir>${project.basedir}</basedir> <docinfo>shared</docinfo> <docinfodir>${project.basedir}/docs/src</docinfodir> @@ -1146,7 +1167,9 @@ limitations under the License. <toc-position>left</toc-position> <stylesdir>${asciidoctor.style.dir}</stylesdir> <stylesheet>tinkerpop.css</stylesheet> - <source-highlighter>coderay</source-highlighter> + <source-highlighter>highlightjs</source-highlighter> + <highlightjsdir>https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0</highlightjsdir> + <highlightjs-languages>groovy</highlightjs-languages> <basedir>${project.basedir}</basedir> <docinfo>shared</docinfo> <docinfodir>${project.basedir}/docs/src</docinfodir> @@ -1173,7 +1196,9 @@ limitations under the License. <toc-position>left</toc-position> <stylesdir>${asciidoctor.style.dir}</stylesdir> <stylesheet>tinkerpop.css</stylesheet> - <source-highlighter>coderay</source-highlighter> + <source-highlighter>highlightjs</source-highlighter> + <highlightjsdir>https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0</highlightjsdir> + <highlightjs-languages>groovy</highlightjs-languages> <basedir>${project.basedir}</basedir> <docinfo>shared</docinfo> <docinfodir>${project.basedir}/docs/src</docinfodir> @@ -1200,7 +1225,9 @@ limitations under the License. <toc-position>left</toc-position> <stylesdir>${asciidoctor.style.dir}</stylesdir> <stylesheet>tinkerpop.css</stylesheet> - <source-highlighter>coderay</source-highlighter> + <source-highlighter>highlightjs</source-highlighter> + <highlightjsdir>https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0</highlightjsdir> + <highlightjs-languages>groovy</highlightjs-languages> <basedir>${project.basedir}</basedir> <docinfo>shared</docinfo> <docinfodir>${project.basedir}/docs/src</docinfodir> @@ -1227,7 +1254,9 @@ limitations under the License. <toc-position>left</toc-position> <stylesdir>${asciidoctor.style.dir}</stylesdir> <stylesheet>tinkerpop.css</stylesheet> - <source-highlighter>coderay</source-highlighter> + <source-highlighter>highlightjs</source-highlighter> + <highlightjsdir>https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0</highlightjsdir> + <highlightjs-languages>groovy</highlightjs-languages> <basedir>${project.basedir}</basedir> <docinfo>shared</docinfo> <docinfodir>${project.basedir}/docs/src</docinfodir> @@ -1254,7 +1283,9 @@ limitations under the License. <toc-position>left</toc-position> <stylesdir>${asciidoctor.style.dir}</stylesdir> <stylesheet>tinkerpop.css</stylesheet> - <source-highlighter>coderay</source-highlighter> + <source-highlighter>highlightjs</source-highlighter> + <highlightjsdir>https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0</highlightjsdir> + <highlightjs-languages>groovy</highlightjs-languages> <basedir>${project.basedir}</basedir> <docinfo>shared</docinfo> <docinfodir>${project.basedir}/docs/src</docinfodir> @@ -1279,7 +1310,9 @@ limitations under the License. <encoding>UTF-8</encoding> <stylesdir>${asciidoctor.style.dir}</stylesdir> <stylesheet>tinkerpop.css</stylesheet> - <source-highlighter>coderay</source-highlighter> + <source-highlighter>highlightjs</source-highlighter> + <highlightjsdir>https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0</highlightjsdir> + <highlightjs-languages>groovy</highlightjs-languages> <basedir>${project.basedir}</basedir> <docinfo>shared</docinfo> <docinfodir>${project.basedir}/docs/src</docinfodir> @@ -1305,7 +1338,9 @@ limitations under the License. <encoding>UTF-8</encoding> <stylesdir>${asciidoctor.style.dir}</stylesdir> <stylesheet>tinkerpop.css</stylesheet> - <source-highlighter>coderay</source-highlighter> + <source-highlighter>highlightjs</source-highlighter> + <highlightjsdir>https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0</highlightjsdir> + <highlightjs-languages>groovy</highlightjs-languages> <basedir>${project.basedir}</basedir> <docinfo>shared</docinfo> <docinfodir>${project.basedir}/docs/src</docinfodir> @@ -1331,7 +1366,9 @@ limitations under the License. <encoding>UTF-8</encoding> <stylesdir>${asciidoctor.style.dir}</stylesdir> <stylesheet>tinkerpop.css</stylesheet> - <source-highlighter>coderay</source-highlighter> + <source-highlighter>highlightjs</source-highlighter> + <highlightjsdir>https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0</highlightjsdir> + <highlightjs-languages>groovy</highlightjs-languages> <basedir>${project.basedir}</basedir> <docinfo>shared</docinfo> <docinfodir>${project.basedir}/docs/src</docinfodir> @@ -1356,7 +1393,9 @@ limitations under the License. <encoding>UTF-8</encoding> <stylesdir>${asciidoctor.style.dir}</stylesdir> <stylesheet>tinkerpop.css</stylesheet> - <source-highlighter>coderay</source-highlighter> + <source-highlighter>highlightjs</source-highlighter> + <highlightjsdir>https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0</highlightjsdir> + <highlightjs-languages>groovy</highlightjs-languages> <basedir>${project.basedir}</basedir> <docinfo>shared</docinfo> <docinfodir>${project.basedir}/docs/src</docinfodir>
