diff --git a/.github/workflows/aot-test.yml b/.github/workflows/aot-test.yml index 9f82281..9271561 100644 --- a/.github/workflows/aot-test.yml +++ b/.github/workflows/aot-test.yml @@ -17,8 +17,9 @@ jobs: - uses: graalvm/setup-graalvm@v1 with: + version: '25.4' java-version: '25' - distribution: 'graalvm' + distribution: 'graalvm-community' github-token: ${{ secrets.GITHUB_TOKEN }} native-image-job-reports: 'true' diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0c19af8..7440fb8 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -16,8 +16,9 @@ jobs: - uses: graalvm/setup-graalvm@v1 with: + version: '25.4' java-version: '25' - distribution: 'graalvm' + distribution: 'graalvm-community' github-token: ${{ secrets.GITHUB_TOKEN }} native-image-job-reports: 'true' diff --git a/README.md b/README.md index 8218a5e..352c513 100644 --- a/README.md +++ b/README.md @@ -70,15 +70,31 @@ jelly-cli rdf from-jelly input.jelly --out-format=ttl > output.ttl You can specify most well-known formats supported by Apache Jena, but also the custom `jelly-text` format. `jelly-text` is a human-readable translation of Jelly binary. It's not meant for machine consumption. It is useful for debugging and inspecting Jelly files. +### Jelly-RDF versions + +`jelly-cli` writes Jelly-RDF 1.2 by default, and reads all versions of Jelly-RDF. Jelly-RDF 1.2 is much faster and better compressed than 1.1, so you are encouraged to upgrade your files to the new version. + +To upgrade your old Jelly-RDF 1.1 file to 1.2, run: + +```shell +jelly-cli rdf to-jelly input.jelly > output.jelly +``` + +If you need files for a reader that only supports Jelly-RDF 1.1, use `--opt.jelly-version=1.1`: + +```shell +jelly-cli rdf to-jelly input.ttl --opt.jelly-version=1.1 > output.jelly +``` + ### Transcode Jelly files -The `rdf transcode` command turns one or more input Jelly streams into a single output stream. It's extremely fast, using a dedicated transcoding algorithm. This algorithm has a limitation: the lookup sizes of the output stream must be greater than or equal to the lookup sizes of the input streams. +The `rdf transcode` command turns one or more input Jelly streams into a single output stream. It's extremely fast, using a dedicated transcoding algorithm. This algorithm has a limitation: the lookup sizes of the output stream must be greater than or equal to the lookup sizes of the input streams. It only works with Jelly-RDF 1.2 streams. ```shell jelly-cli rdf transcode input.jelly > output.jelly ``` -If you need other options than that, use `rdf to-jelly` with Jelly input instead. It fully decodes and re-encodes the stream, so it's slower, but you can set any `--opt.*` options. The same works for Jelly-SPARQL with `sparql to-jelly --in-format jelly-sparql`. +If you need other options than that, or want to turn a Jelly-RDF 1.1 file into 1.2, use `rdf to-jelly` with Jelly input instead. It fully decodes and re-encodes the stream, so it's slower, but you can set any `--opt.*` options. The same works for Jelly-SPARQL with `sparql to-jelly --in-format jelly-sparql`. ```shell jelly-cli rdf to-jelly input.jelly --opt.max-name-table-size=256 > output.jelly diff --git a/build.sbt b/build.sbt index b002862..e366dad 100644 --- a/build.sbt +++ b/build.sbt @@ -11,7 +11,7 @@ resolvers += "Sonatype OSS Snapshots" at "https://central.sonatype.com/repository/maven-snapshots" lazy val jenaV = "6.2.0" -lazy val jellyV = "3.7.3+80-0f51136f-SNAPSHOT" +lazy val jellyV = "3.7.3+85-45406b30-SNAPSHOT" lazy val graalvmV = "25.4.4.1.1" addCommandAlias("fixAll", "scalafixAll; scalafmtAll") @@ -23,8 +23,8 @@ def isDevBuild: Boolean = lazy val graalOptions = Seq( // Do a fast build if it's a dev build - // For the release build, optimize for speed and make a build report - if (isDevBuild) Seq("-Ob") else Seq("-O3", "--emit build-report"), + // For the release build, optimize for speed + if (isDevBuild) Seq("-Ob") else Seq("-O3"), ).flatten ++ Seq( // If using dependencies on Scala <3.8 and JDK >=24, we need to allow unsafe memory access. // Otherwise, we get annoying warnings on startup. @@ -42,7 +42,6 @@ lazy val graalOptions = Seq( "--initialize-at-build-time=org.glassfish.json.UnicodeDetectingInputStream", "-H:+TrackPrimitiveValues", // SkipFlow optimization "-H:+UsePredicates", // SkipFlow optimization - "-H:+MLCallCountProfileInference", // ML inference for hot/cold method detection // Make sure we don't include stuff that should be unreachable in the native image "-H:AbortOnMethodReachable=*UUID.randomUUID*", // Include XML error messages diff --git a/src/main/resources/META-INF/native-image/com.github.ben-manes.caffeine/caffeine/reflect-config.json b/src/main/resources/META-INF/native-image/com.github.ben-manes.caffeine/caffeine/reflect-config.json index feb2266..14538cb 100644 --- a/src/main/resources/META-INF/native-image/com.github.ben-manes.caffeine/caffeine/reflect-config.json +++ b/src/main/resources/META-INF/native-image/com.github.ben-manes.caffeine/caffeine/reflect-config.json @@ -327,6 +327,11 @@ "boolean" ] } + ], + "fields": [ + { + "name": "FACTORY" + } ] }, { @@ -343,6 +348,11 @@ "boolean" ] } + ], + "fields": [ + { + "name": "FACTORY" + } ] }, { @@ -356,6 +366,11 @@ "boolean" ] } + ], + "fields": [ + { + "name": "FACTORY" + } ] }, { @@ -372,6 +387,11 @@ "boolean" ] } + ], + "fields": [ + { + "name": "FACTORY" + } ] }, { @@ -388,6 +408,11 @@ "boolean" ] } + ], + "fields": [ + { + "name": "FACTORY" + } ] }, { @@ -404,6 +429,11 @@ "boolean" ] } + ], + "fields": [ + { + "name": "FACTORY" + } ] }, { @@ -420,6 +450,11 @@ "boolean" ] } + ], + "fields": [ + { + "name": "FACTORY" + } ] }, { @@ -436,6 +471,11 @@ "boolean" ] } + ], + "fields": [ + { + "name": "FACTORY" + } ] }, { @@ -452,6 +492,11 @@ "boolean" ] } + ], + "fields": [ + { + "name": "FACTORY" + } ] }, { @@ -468,6 +513,11 @@ "boolean" ] } + ], + "fields": [ + { + "name": "FACTORY" + } ] }, { @@ -484,6 +534,11 @@ "boolean" ] } + ], + "fields": [ + { + "name": "FACTORY" + } ] }, { @@ -522,6 +577,11 @@ "boolean" ] } + ], + "fields": [ + { + "name": "FACTORY" + } ] }, { @@ -538,6 +598,11 @@ "boolean" ] } + ], + "fields": [ + { + "name": "FACTORY" + } ] }, { @@ -554,6 +619,11 @@ "boolean" ] } + ], + "fields": [ + { + "name": "FACTORY" + } ] }, { diff --git a/src/main/scala/eu/neverblink/jelly/cli/JellyCommand.scala b/src/main/scala/eu/neverblink/jelly/cli/JellyCommand.scala index 5572cd1..0a5153e 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/JellyCommand.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/JellyCommand.scala @@ -120,6 +120,8 @@ abstract class JellyCommand[T <: HasJellyCommandOptions: {Parser, Help}] extends else System.in final def setStdIn(data: InputStream): Unit = + // Entering the test mode resets the input, so do it before setting it + if !isTest then testMode(true) in = data final def getOutStream: OutputStream = diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfFromJelly.scala b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfFromJelly.scala index e1871bf..921ed34 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfFromJelly.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfFromJelly.scala @@ -6,7 +6,7 @@ import eu.neverblink.jelly.cli.command.rdf.util.* import eu.neverblink.jelly.cli.command.rdf.util.RdfFormat.* import eu.neverblink.jelly.cli.util.args.IndexRange import eu.neverblink.jelly.cli.util.io.ProtoText -import eu.neverblink.jelly.cli.util.jena.riot.JellyWriterUtil +import eu.neverblink.jelly.cli.util.jena.riot.{JellyMessageWriter, JellyWriterUtil} import eu.neverblink.jelly.cli.util.jena.{ JenaSystemOptions, RdfSparqlConverter, @@ -16,7 +16,7 @@ import eu.neverblink.jelly.cli.util.jena.{ import eu.neverblink.jelly.convert.jena.JenaConverterFactory import eu.neverblink.jelly.core.JellyOptions import eu.neverblink.jelly.core.RdfHandler.AnyStatementHandler -import eu.neverblink.jelly.core.proto.v1.RdfStreamFrame +import eu.neverblink.jelly.core.proto.v1.{RdfStreamFrame, RdfStreamType} import eu.neverblink.jelly.core.sparql.JellySparqlConstants import org.apache.jena.graph.{Node, Triple} import org.apache.jena.riot.system.StreamRDF @@ -25,7 +25,6 @@ import org.apache.jena.sparql.core.Quad import java.io.{InputStream, OutputStream} import java.nio.charset.StandardCharsets.UTF_8 -import scala.jdk.CollectionConverters.* object RdfFromJellyPrint extends RdfCommandPrintUtil[RdfFormat.Writeable]: override val defaultFormat: RdfFormat = RdfFormat.NQuads @@ -87,7 +86,7 @@ object RdfFromJelly extends RdfSerDesCommand[RdfFromJellyOptions, RdfFormat.Writ val defaultAction: WriteAction = (in, out, opt) => jellyToLang( - in, + JellyUtil.iterateRdfStream(in), StreamRDFWriter.getWriterStream(out, RdfFormat.NQuads.jenaLang), RdfFormat.NQuads, opt, @@ -111,16 +110,23 @@ object RdfFromJelly extends RdfSerDesCommand[RdfFromJellyOptions, RdfFormat.Writ case (RdfFormat.JellyBinary, _) => Some(jellyToJelly) case (j: RdfFormat.Jena.StreamWriteable, _) => Some((in, out, opt) => - jellyToLang(in, StreamRDFWriter.getWriterStream(out, j.jenaLang), j, opt), + jellyToLang( + JellyUtil.iterateRdfStream(in), + StreamRDFWriter.getWriterStream(out, j.jenaLang), + j, + opt, + ), ) case (j: RdfFormat.Jena.BatchWriteable, true) => Some((in, out, opt) => StreamRdfCombiningBatchWriter(out, j.jenaLang).runAndOutput(x => - jellyToLang(in, x, j, opt), + jellyToLang(JellyUtil.iterateRdfStream(in), x, j, opt), ), ) case (j: RdfFormat.Jena.BatchWriteable, false) => - Some((in, out, opt) => jellyToLang(in, StreamRdfBatchWriter(out, j.jenaLang), j, opt)) + Some((in, out, opt) => + jellyToLang(JellyUtil.iterateRdfStream(in), StreamRdfBatchWriter(out, j.jenaLang), j, opt), + ) case (RdfFormat.JellyText, _) => Some(jellyBinaryToText) case (RdfFormat.JellySparql, _) => Some(jellyToSparql) @@ -132,15 +138,30 @@ object RdfFromJelly extends RdfSerDesCommand[RdfFromJellyOptions, RdfFormat.Writ outputStream: OutputStream, opt: RdfFromJellyOptions, ): Unit = - val writer = JellyWriterUtil.createWriter( - RdfJellySerializationOptions.defaultFor(RdfFormat.JellyBinary), - RdfJellySerializationOptions.defaultRowsPerFrame, - enableNamespaceDeclarations = false, - delimited = true, - outputStream, - ) - writer.start() - jellyToLang(inputStream, writer, RdfFormat.JellyBinary, opt) + val frames = JellyUtil.iterateRdfStream(inputStream).buffered + val options = RdfJellySerializationOptions.defaultOptions + val frameSize = RdfJellySerializationOptions.defaultRowsPerFrame(rowLayout = false) + JellyUtil.firstOptions(frames).filter(_.getStreamType == RdfStreamType.MESSAGES) match + case Some(inputOptions) => + val writer = JellyMessageWriter( + options.clone() + .setPhysicalType(inputOptions.getPhysicalType) + .setStreamType(RdfStreamType.MESSAGES), + frameSize, + delimited = true, + outputStream, + ) + jellyToLang(frames, writer, RdfFormat.JellyBinary, opt, writer.endMessage) + case None => + val writer = JellyWriterUtil.createWriter( + options, + frameSize, + enableNamespaceDeclarations = false, + delimited = true, + outputStream, + ) + writer.start() + jellyToLang(frames, writer, RdfFormat.JellyBinary, opt) /** Converts the Jelly-RDF stream to a Jelly-SPARQL result set of ?s ?p ?o (?g), with the default * options. To set them, use sparql to-jelly --in-format jelly-rdf. @@ -162,10 +183,11 @@ object RdfFromJelly extends RdfSerDesCommand[RdfFromJellyOptions, RdfFormat.Writ * stream */ private def jellyToLang( - inputStream: InputStream, + frames: Iterator[RdfStreamFrame], writer: StreamRDF, format: RdfFormat, options: RdfFromJellyOptions, + onMessageEnd: () => Unit = () => (), ): Unit = // Whether the output is active at this moment var outputEnabled = false @@ -174,6 +196,8 @@ object RdfFromJelly extends RdfSerDesCommand[RdfFromJellyOptions, RdfFormat.Writ if outputEnabled then writer.prefix(prefix, namespace.getURI) } + override def handleMessageEnd(): Unit = if outputEnabled then onMessageEnd() + override def handleTriple(subject: Node, predicate: Node, `object`: Node): Unit = { if outputEnabled then writer.triple(Triple.create(subject, predicate, `object`)) } @@ -201,19 +225,19 @@ object RdfFromJelly extends RdfSerDesCommand[RdfFromJellyOptions, RdfFormat.Writ ) val inputFrames = takeFrames.end match - case Some(end) => JellyUtil.iterateRdfStream(inputStream).take(end) - case None => JellyUtil.iterateRdfStream(inputStream) + case Some(end) => frames.take(end) + case None => frames val startFrom = takeFrames.start.getOrElse(0) for (frame, i) <- inputFrames.zipWithIndex do // If we are not yet in the output range, still fully parse the frame and update the decoder // state. We need this to decode the later frames correctly. - if i < startFrom then for row <- frame.getRows.asScala do decoder.ingestRow(row) + if i < startFrom then decoder.ingestFrame(frame) else // TODO: write frame index as a comment here // https://github.com/Jelly-RDF/cli/issues/4 outputEnabled = true // We are in the output range, so we can start writing the output - for row <- frame.getRows.asScala do decoder.ingestRow(row) + decoder.ingestFrame(frame) writer.finish() /** This method reads the Jelly file, rewrites it to Jelly text and writes it to some output diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfInspect.scala b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfInspect.scala index 180f0f8..1a5ff2e 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfInspect.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfInspect.scala @@ -11,9 +11,8 @@ import scala.jdk.CollectionConverters.* import java.io.InputStream @HelpMessage( "Prints statistics about a Jelly-RDF stream.\n" + - "Statistics include: Jelly stream options and counts/sizes of various row types, " + - "including triples, quads, names, prefixes, " + - "namespaces, datatypes, and graphs.\n" + + "Statistics include: Jelly stream options and counts of statements and the different types " + + "of RDF terms.\n" + "Output statistics are returned as a valid YAML. \n" + "If no input file is specified, the input is read from stdin.\n" + "If no output file is specified, the output is written to stdout.\n" + @@ -77,21 +76,25 @@ object RdfInspect extends JellyCommand[RdfInspectOptions]: detail: Boolean, )(using FrameInfo.StatisticCollector): (RdfStreamOptions, Iterator[FrameInfo]) = - inline def computeMetrics( - frame: RdfStreamFrame, - frameIndex: Int, - ): FrameInfo = - val metadata = frame.getMetadata.asScala.map(entry => entry.getKey -> entry.getValue).toMap - val metrics = - if detail then new FrameDetailInfo(frameIndex, metadata) - else FrameInfo(frameIndex, metadata) - metrics.processFrame(frame) - metrics - try { val allRows = JellyUtil.iterateRdfStream(inputStream).buffered // we need to check if the first frame contains options val streamOptions = checkOptions(allRows.headOption) + val rowLayout = StreamOptionsUtil.isRowLayout(streamOptions) + val messages = streamOptions.getStreamType == RdfStreamType.MESSAGES + + def computeMetrics( + frame: RdfStreamFrame, + frameIndex: Int, + ): FrameInfo = + val metadata = frame.getMetadata.asScala.map(entry => entry.getKey -> entry.getValue).toMap + val metrics = + if !rowLayout then ColumnFrameInfo(frameIndex, metadata, messages, detail) + else if detail then new FrameDetailInfo(frameIndex, metadata) + else FrameInfo(frameIndex, metadata) + metrics.processFrame(frame) + metrics + // We compute the metrics for each frame // and then sum them all during the printing if desired val frameIterator = allRows.zipWithIndex.map { case (maybeFrame, frameIndex) => diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJelly.scala b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJelly.scala index 32d7e57..ad0e465 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJelly.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJelly.scala @@ -7,11 +7,16 @@ import eu.neverblink.jelly.cli.command.rdf.util.RdfFormat.* import eu.neverblink.jelly.cli.util.io.ProtoText import eu.neverblink.jelly.cli.util.jena.{JenaSystemOptions, RdfSparqlConverter} import eu.neverblink.jelly.cli.util.jena.riot.{JellyWriterUtil, RiotParserUtil} -import eu.neverblink.jelly.core.{JellyOptions, RdfProtoDeserializationError} +import eu.neverblink.jelly.core.{ + JellyOptions, + RdfProtoDeserializationError, + RdfProtoSerializationError, +} import eu.neverblink.jelly.core.proto.v1.* import java.io.{BufferedReader, InputStream, InputStreamReader, OutputStream} import java.nio.charset.StandardCharsets.UTF_8 +import scala.collection.mutable.ArrayBuffer import scala.util.Using object RdfToJellyPrint extends RdfCommandPrintUtil[RdfFormat.Readable]: @@ -27,8 +32,8 @@ object RdfToJellyPrint extends RdfCommandPrintUtil[RdfFormat.Readable]: "variables, which are taken by position, not by name: subject, predicate, object, and graph. " + "Each solution becomes one statement. With 3 variables, the output is a TRIPLES stream, " + "with 4 a QUADS stream, where an unbound graph means the default graph. Each result set of " + - "a PUNCTUATED stream becomes one frame, and the logical type is GRAPHS or DATASETS, unless " + - "set with --opt.logical-type.\n" + + "a PUNCTUATED stream becomes one RDF Message (stream type messages), unless the stream type " + + "is set with --opt.stream-type.\n" + "Note: this command works in a streaming manner and scales well to large files. ", ) @ArgsName("") @@ -52,10 +57,14 @@ case class RdfToJellyOptions( ) optionsFrom: Option[String] = None, @HelpMessage( - "Target number of rows per frame – the writer may slightly exceed that. Default: " + - RdfJellySerializationOptions.defaultRowsPerFrame, + "Target number of statements per frame (in Jelly-RDF 1.1: rows, and the writer may " + + "slightly exceed that). Default: " + + RdfJellySerializationOptions.defaultRowsPerFrame(rowLayout = + false, + ) + " (in Jelly-RDF 1.1: " + + RdfJellySerializationOptions.defaultRowsPerFrame(rowLayout = true) + ")", ) - rowsPerFrame: Int = RdfJellySerializationOptions.defaultRowsPerFrame, + rowsPerFrame: Option[Int] = None, @HelpMessage( "Whether to preserve explicit namespace declarations in the output (PREFIX: in Turtle). " + "Default: false", @@ -68,7 +77,10 @@ case class RdfToJellyOptions( delimited: Boolean = true, @Recurse rdfPerformanceOptions: RdfPerformanceOptions = RdfPerformanceOptions(), -) extends HasJellyCommandOptions +) extends HasJellyCommandOptions: + def frameSize: Int = rowsPerFrame.getOrElse( + RdfJellySerializationOptions.defaultRowsPerFrame(jellySerializationOptions.isRowLayout), + ) object RdfToJelly extends RdfSerDesCommand[RdfToJellyOptions, RdfFormat.Readable]: @@ -123,7 +135,8 @@ object RdfToJelly extends RdfSerDesCommand[RdfToJellyOptions, RdfFormat.Readable inputStream, outputStream, getOptions.jellySerializationOptions.asRdfStreamOptions, - getOptions.rowsPerFrame, + streamTypeSet = getOptions.jellySerializationOptions.`opt.streamType`.isDefined, + getOptions.frameSize, getOptions.delimited, ) @@ -142,7 +155,8 @@ object RdfToJelly extends RdfSerDesCommand[RdfToJellyOptions, RdfFormat.Readable opt: RdfToJellyOptions, ): Unit = val jellyOpt = getOptions.jellySerializationOptions.asRdfStreamOptions - if jellyOpt.getPhysicalType == PhysicalStreamType.UNSPECIFIED && !isQuietMode && + if StreamOptionsUtil.isRowLayout(jellyOpt) && + jellyOpt.getPhysicalType == PhysicalStreamType.UNSPECIFIED && !isQuietMode && isLogicalGrouped(jellyOpt) then printLine( @@ -153,19 +167,26 @@ object RdfToJelly extends RdfSerDesCommand[RdfToJellyOptions, RdfFormat.Readable ) val jellyWriter = JellyWriterUtil.createWriter( jellyOpt, - getOptions.rowsPerFrame, + getOptions.frameSize, getOptions.enableNamespaceDeclarations, getOptions.delimited, outputStream, ) - RiotParserUtil.parse( - getOptions.rdfPerformanceOptions.resolveIris, - format, - inputStream, - jellyWriter, - ) - jellyWriter.finish() + try + RiotParserUtil.parse( + getOptions.rdfPerformanceOptions.resolveIris, + format, + inputStream, + jellyWriter, + ) + jellyWriter.finish() + catch + // Generalized statements and RDF-star can only be written in Jelly-RDF 1.1 + case e: RdfProtoSerializationError if e.getMessage.contains("Jelly-RDF 1.1") => + throw JellySerializationError( + e.getMessage + " Use --opt.jelly-version=1.1 to write Jelly-RDF 1.1.", + ) /** Convert Jelly text to Jelly binary. * @param inputStream @@ -196,24 +217,26 @@ object RdfToJelly extends RdfSerDesCommand[RdfToJellyOptions, RdfFormat.Readable } } + /** Warns about logical types that do not fit the physical type (only Jelly-RDF 1.1). */ private def checkAndWarnTypeCombination(): Unit = { val rdfStreamOptions = getOptions.jellySerializationOptions.asRdfStreamOptions val physicalType = rdfStreamOptions.getPhysicalType val logicalType = rdfStreamOptions.getLogicalType - try { - // This check will find physical/logical clashes, since all other fields are checked with <=, - // so they pass when compared against themselves - JellyOptions.checkCompatibility(rdfStreamOptions, rdfStreamOptions) - } catch { - case _: RdfProtoDeserializationError => - printLine( - s"WARNING: Selected combination of logical/physical stream types ($logicalType/$physicalType) is unsupported. " + - "Use --quiet to silence this warning.", - true, - ) - } + if StreamOptionsUtil.isRowLayout(rdfStreamOptions) then + try { + // This check will find physical/logical clashes, since all other fields are checked with <=, + // so they pass when compared against themselves + JellyOptions.checkCompatibility(rdfStreamOptions, rdfStreamOptions) + } catch { + case _: RdfProtoDeserializationError => + printLine( + s"WARNING: Selected combination of logical/physical stream types ($logicalType/$physicalType) is unsupported. " + + "Use --quiet to silence this warning.", + true, + ) + } } /** Check if the logical type is defined and grouped. @@ -256,5 +279,20 @@ object RdfToJelly extends RdfSerDesCommand[RdfToJellyOptions, RdfFormat.Readable // The only options we can respect in this mode are the frame size and the delimited flag // The others are ignored, because we are doing a 1:1 conversion - if getOptions.delimited then rows.grouped(getOptions.rowsPerFrame).map(_.mkString("\n")) - else Iterator(rows.mkString("\n")) + if !getOptions.delimited then Iterator(rows.mkString("\n")) + else + // Rows are grouped into frames of the given size. A Jelly-RDF 1.2 frame ends with its + // column batch, so the rows (stream options) before it go into the same frame. + val frameSize = getOptions.rowsPerFrame + .getOrElse(RdfJellySerializationOptions.defaultRowsPerFrame(rowLayout = true)) + val frame = ArrayBuffer[String]() + def isColumns(block: String) = + block.linesIterator.find(!_.startsWith("#")).exists(_.startsWith("columns")) + rows.flatMap { block => + frame += block + if isColumns(block) || frame.size >= frameSize then + val text = frame.mkString("\n") + frame.clear() + Some(text) + else None + } ++ Iterator.single(()).flatMap(_ => Option.when(frame.nonEmpty)(frame.mkString("\n"))) diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfTranscode.scala b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfTranscode.scala index 4382d9d..5330b6d 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfTranscode.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfTranscode.scala @@ -3,11 +3,10 @@ package eu.neverblink.jelly.cli.command.rdf import caseapp.* import eu.neverblink.jelly.cli.* import eu.neverblink.jelly.cli.command.rdf.util.* -import eu.neverblink.jelly.core.proto.v1.{LogicalStreamType, RdfStreamOptions} +import eu.neverblink.jelly.core.proto.v1.RdfStreamOptions import eu.neverblink.jelly.core.{JellyOptions, JellyTranscoderFactory, RdfProtoTranscodingError} import java.io.{InputStream, OutputStream} -import scala.jdk.CollectionConverters.* @HelpMessage( "Quickly transcodes the input Jelly file into another Jelly file.\n" + @@ -15,6 +14,8 @@ import scala.jdk.CollectionConverters.* "The input may be a concatenation of multiple Jelly streams.\n" + "Currently only frame-by-frame transcoding is supported.\n" + "The output's options must be greater than or equal to the input options.\n" + + "Only Jelly-RDF 1.2 streams can be transcoded. To turn a Jelly-RDF 1.1 stream into 1.2, " + + "use rdf to-jelly with Jelly input, which fully decodes and re-encodes it.\n" + "Note: this command works in a streaming manner and scales well to large files.", ) @ArgsName("") @@ -40,6 +41,12 @@ object RdfTranscode extends JellyCommand[RdfTranscodeOptions]: override final def group = "rdf" override def doRun(options: RdfTranscodeOptions, remainingArgs: RemainingArgs): Unit = + if options.jellySerializationOptions.isRowLayout then + throw InvalidArgument( + "--opt.jelly-version", + "1.1", + Some("rdf transcode only writes Jelly-RDF 1.2"), + ) val outOpt = options.jellySerializationOptions.asRdfStreamOptions val (inputStream, outputStream) = getIoStreamsFromOptions(remainingArgs.remaining.headOption, options.outputFile) @@ -60,22 +67,20 @@ object RdfTranscode extends JellyCommand[RdfTranscodeOptions]: outOpt: RdfStreamOptions, ): Unit = val in = JellyUtil.iterateRdfStream(inputStream).buffered - val head = in.head - if head.getRows.asScala.isEmpty then throw CriticalException("Empty input stream") - if !head.getRows.asScala.head.hasOptions then - throw CriticalException("First input row is not an options row") - val inOpt = head.getRows.asScala.head.getOptions + val inOpt = JellyUtil.requireFirstOptions(in) + if StreamOptionsUtil.isRowLayout(inOpt) then + throw CriticalException( + "The input is a Jelly-RDF 1.1 stream, but rdf transcode only supports Jelly-RDF 1.2. " + + "To turn it into Jelly-RDF 1.2, use rdf to-jelly with Jelly input instead, " + + "for example: rdf to-jelly --in-format jelly", + ) val transcoder = JellyTranscoderFactory.fastMergingTranscoder( JellyOptions.DEFAULT_SUPPORTED_OPTIONS, outOpt.clone() - // There is no way to specify the physical type with options currently. - // Just use the one from the input. + // The physical type and the stream type must be the same as in the input .setPhysicalType(inOpt.getPhysicalType) - .setLogicalType( - if outOpt.getLogicalType == LogicalStreamType.UNSPECIFIED then inOpt.getLogicalType - else outOpt.getLogicalType, - ), + .setStreamType(inOpt.getStreamType), ) in.map(transcoder.ingestFrame) diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfValidate.scala b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfValidate.scala index a5cf72a..4217789 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfValidate.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfValidate.scala @@ -116,14 +116,12 @@ object RdfValidate extends JellyCommand[RdfValidateOptions]: throw CriticalException("Expected undelimited input, but the file was delimited") private def validateOptions(frames: Seq[RdfStreamFrame]): RdfStreamOptions = - if !frames.head.getRows.asScala.head.hasOptions then - throw CriticalException("First row in the input stream does not contain stream options") - val streamOptions = frames.head.getRows.asScala.head.getOptions + val streamOptions = JellyUtil.firstOptions(frames.iterator.buffered).getOrElse( + throw CriticalException("First row in the input stream does not contain stream options"), + ) // If we have expected options, we need to read and validate them val expectedOptions = getOptions.optionsFile.map { optionsFileName => - val o = Using.resource(IoUtil.inputStream(optionsFileName)) { is => - JellyUtil.iterateRdfStream(is).next().getRows.asScala.head.getOptions - } + val o = JellyUtil.loadOptionsFromFile(optionsFileName) if streamOptions != o then throw CriticalException( s"Stream options do not match the expected options in $optionsFileName\n" + @@ -162,9 +160,9 @@ object RdfValidate extends JellyCommand[RdfValidateOptions]: override def handleTriple(subject: Node, predicate: Node, `object`: Node): Unit = { val t = Triple.create(subject, predicate, `object`) - if !opt.getGeneralizedStatements && StatementUtils.isGeneralized(t) then + if !StreamOptionsUtil.allowsGeneralized(opt) && StatementUtils.isGeneralized(t) then throw CriticalException(s"Unexpected generalized triple in frame $currentPosition: $t") - if !opt.getRdfStar && StatementUtils.hasTripleTerms(t) then + if !StreamOptionsUtil.allowsTripleTerms(opt) && StatementUtils.hasTripleTerms(t) then throw CriticalException( s"Unexpected triple term in triple in frame $currentPosition: $t", ) @@ -174,9 +172,9 @@ object RdfValidate extends JellyCommand[RdfValidateOptions]: override def handleQuad(subject: Node, predicate: Node, `object`: Node, graph: Node): Unit = { val q = Quad.create(graph, subject, predicate, `object`) - if !opt.getGeneralizedStatements && StatementUtils.isGeneralized(q) then + if !StreamOptionsUtil.allowsGeneralized(opt) && StatementUtils.isGeneralized(q) then throw CriticalException(s"Unexpected generalized quad in frame $currentPosition: $q") - if !opt.getRdfStar && StatementUtils.hasTripleTerms(q) then + if !StreamOptionsUtil.allowsTripleTerms(opt) && StatementUtils.hasTripleTerms(q) then throw CriticalException(s"Unexpected triple term in quad in frame $currentPosition: $q") // Add the quad to the comparison set, if we are in the compare range if currentPosition >= startFrom then jellyStreamConsumer.quad(q) @@ -190,16 +188,20 @@ object RdfValidate extends JellyCommand[RdfValidateOptions]: case Some(end) => frames.take(end) case None => frames + val rowLayout = StreamOptionsUtil.isRowLayout(opt) for (frame, i) <- frames2.zipWithIndex do - for row <- frame.getRows.asScala do - if row.hasOptions && row.getOptions != opt then - throw CriticalException( - s"Later occurrence of stream options in frame $i does not match the first", - ) - // Push the stream frames through the decoder - // This will catch most of the errors - currentPosition = i - dec.ingestRow(row) + // In Jelly-RDF 1.2, the options may be restated with other values, and the decoder checks + // that they are valid + if rowLayout then + for row <- frame.getRows.asScala do + if row.hasOptions && row.getOptions != opt then + throw CriticalException( + s"Later occurrence of stream options in frame $i does not match the first", + ) + // Push the stream frames through the decoder + // This will catch most of the errors + currentPosition = i + dec.ingestFrame(frame) // Compare the Jelly data with the reference RDF data, if specified maybeRdfComparison.foreach { rdfComparison => val actual = jellyStreamConsumer.asInstanceOf[StreamRdfCollector] @@ -221,7 +223,7 @@ object RdfValidate extends JellyCommand[RdfValidateOptions]: frames: Seq[RdfStreamFrame], ): Seq[RdfStreamFrame] = if frames.isEmpty then throw CriticalException("Empty input stream") - else if frames.head.getRows.isEmpty then + else if frames.head.getRows.isEmpty && frames.head.getColumns == null then // We want to accept empty frames in the stream, but not empty streams if frames.tail.isEmpty then throw CriticalException("All frames are empty") skipEmptyFrames(frames.tail) diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/ColumnInfo.scala b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/ColumnInfo.scala new file mode 100644 index 0000000..0fbbcdc --- /dev/null +++ b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/ColumnInfo.scala @@ -0,0 +1,157 @@ +package eu.neverblink.jelly.cli.command.rdf.util + +import com.google.protobuf.ByteString +import eu.neverblink.jelly.core.proto.v1.{RdfColumn, RdfStreamFrame} + +import scala.jdk.CollectionConverters.* + +/** Statistics of the values in columns, used by Jelly-RDF 1.2 and Jelly-SPARQL. */ +final class ColumnInfo(using statCollector: FrameInfo.StatisticCollector): + private object stat: + var iri: Long = 0 + var bnode: Long = 0 + var literal: Long = 0 + var tripleTerm: Long = 0 + var layout: Long = 0 + + private def measureValues(count: Int, size: => Long): Long = + if statCollector == FrameInfo.CountStatistic then count else size + + def process(c: RdfColumn): Unit = + stat.iri += measureValues( + c.getNameIds.size, + c.getNameIds.computeUInt32SizeNoTag + c.getPrefixIds.computeUInt32SizeNoTag, + ) + stat.literal += measureValues( + c.getLexValues.size, + c.getLexValues.computeStringSizeNoTag + c.getLiteralKinds.computeUInt32SizeNoTag + + c.getLangtags.computeStringSizeNoTag + c.getLangtagDirections.computeUInt32SizeNoTag, + ) + stat.bnode += measureValues(c.getBnodes.size, c.getBnodes.computeStringSizeNoTag) + stat.tripleTerm += measureValues( + c.getTripleTerms.size, + c.getTripleTerms.asScala.iterator.map(statCollector.measure).sum, + ) + stat.layout += measureValues( + c.getLayouts.size, + c.getLayouts.computeUInt32SizeNoTag + c.getKinds.size, + ) + + def +=(other: ColumnInfo): ColumnInfo = + stat.iri += other.stat.iri + stat.bnode += other.stat.bnode + stat.literal += other.stat.literal + stat.tripleTerm += other.stat.tripleTerm + stat.layout += other.stat.layout + this + + /** All values, without the layout. */ + def total: Long = stat.iri + stat.bnode + stat.literal + stat.tripleTerm + + def format(): Seq[(String, Long)] = + val name = statCollector.name() + Seq( + "iri_value_" + name -> stat.iri, + "bnode_value_" + name -> stat.bnode, + "literal_value_" + name -> stat.literal, + "triple_term_value_" + name -> stat.tripleTerm, + "layout_" + name -> stat.layout, + ) + +end ColumnInfo + +/** Statistics of a frame of a Jelly-RDF 1.2 stream (column layout), or of several frames added + * together. + * + * @param messages + * whether the stream is a stream of RDF Messages, so that the message boundaries are counted + */ +class ColumnFrameInfo( + frameIndex: Long, + metadata: Map[String, ByteString], + val messages: Boolean, + val detail: Boolean, +)(using + statCollector: FrameInfo.StatisticCollector, +) extends FrameInfo(frameIndex, metadata): + private object stat: + var frame: Long = 0 + var statement: Long = 0 + var option: Long = 0 + var namespace: Long = 0 + var name: Long = 0 + var prefix: Long = 0 + var datatype: Long = 0 + var messageEnd: Long = 0 + private object column: + val subject: ColumnInfo = ColumnInfo() + val predicate: ColumnInfo = ColumnInfo() + val `object`: ColumnInfo = ColumnInfo() + val graph: ColumnInfo = ColumnInfo() + def all: Seq[(String, ColumnInfo)] = + Seq("subject" -> subject, "predicate" -> predicate, "object" -> `object`, "graph" -> graph) + + override def +=(other: FrameInfo): FrameInfo = + frameCount += 1 + other match + case o: ColumnFrameInfo => + stat.frame += o.stat.frame + stat.statement += o.stat.statement + stat.option += o.stat.option + stat.namespace += o.stat.namespace + stat.name += o.stat.name + stat.prefix += o.stat.prefix + stat.datatype += o.stat.datatype + stat.messageEnd += o.stat.messageEnd + for ((_, mine), (_, theirs)) <- column.all.zip(o.column.all) do mine += theirs + case _ => + this + + override def processFrame(frame: RdfStreamFrame): Unit = + stat.frame += statCollector.measure(frame) + frame.getRows.asScala.filter(_.hasOptions).foreach { row => + stat.option += statCollector.measure(row.getOptions) + } + Option(frame.getColumns).foreach { batch => + stat.statement += batch.getRowCount + stat.namespace += batch.getNamespaces.asScala.iterator.map(statCollector.measure).sum + stat.name += FrameInfo.measureLookup(batch.getNames) + stat.prefix += FrameInfo.measureLookup(batch.getPrefixes) + stat.datatype += FrameInfo.measureLookup(batch.getDatatypes) + stat.messageEnd += + (if statCollector == FrameInfo.CountStatistic then batch.getMessageLengths.size + else batch.getMessageLengths.computeUInt32SizeNoTag) + Option(batch.getSubjects).foreach(column.subject.process) + Option(batch.getPredicates).foreach(column.predicate.process) + Option(batch.getObjects).foreach(column.`object`.process) + Option(batch.getGraphs).foreach(column.graph.process) + } + + /** The values of all columns, by kind. */ + def valuesByKind: ColumnInfo = + val sum = ColumnInfo() + column.all.foreach((_, c) => sum += c) + sum + + /** The values of each column, by kind. */ + def valuesByColumn: Seq[(String, ColumnInfo)] = column.all + + /** The number or size of all values in each column. */ + def formatTotalsByColumn(): Seq[(String, Long)] = + column.all.map((name, info) => s"${name}_${statCollector.name()}" -> info.total) + + override def format(): Seq[(String, Long)] = + val name = statCollector.name() + Seq( + "frame_" + name -> stat.frame, + // The number of statements, whatever the statistic + "statement_count" -> stat.statement, + "option_" + name -> stat.option, + "namespace_" + name -> stat.namespace, + "name_" + name -> stat.name, + "prefix_" + name -> stat.prefix, + "datatype_" + name -> stat.datatype, + ) ++ valuesByKind.format() ++ + (if messages then Seq("message_end_" + name -> stat.messageEnd) else Seq()) + +end ColumnFrameInfo diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/JellyUtil.scala b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/JellyUtil.scala index 314ca15..7615d0d 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/JellyUtil.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/JellyUtil.scala @@ -1,5 +1,6 @@ package eu.neverblink.jelly.cli.command.rdf.util +import eu.neverblink.jelly.cli.CriticalException import eu.neverblink.jelly.cli.util.io.IoUtil import eu.neverblink.jelly.core.utils.IoUtils import eu.neverblink.jelly.core.proto.v1.{RdfStreamFrame, RdfStreamOptions} @@ -41,6 +42,21 @@ object JellyUtil: (false, Iterator(RdfStreamFrame.parseFrom(delimitingResponse.newInput))) } + /** The stream options in the first row of the stream, if it is an options row. */ + def firstOptions(frames: collection.BufferedIterator[RdfStreamFrame]): Option[RdfStreamOptions] = + frames.headOption.flatMap(f => Option.when(!f.getRows.isEmpty)(f.getRows.iterator.next())) + .filter(_.hasOptions).map(_.getOptions) + + /** The stream options in the first row of the stream. + * + * @throws CriticalException + * if the stream is empty, or does not start with an options row + */ + def requireFirstOptions(frames: collection.BufferedIterator[RdfStreamFrame]): RdfStreamOptions = + if !frames.hasNext || frames.head.getRows.isEmpty && frames.head.getColumns == null then + throw CriticalException("Empty input stream") + firstOptions(frames).getOrElse(throw CriticalException("First input row is not an options row")) + /** Reads the stream options from the first row of a Jelly-RDF file. * @param fileName * file to read diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/MetricsPrinter.scala b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/MetricsPrinter.scala index fc41c70..c6126b8 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/MetricsPrinter.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/MetricsPrinter.scala @@ -4,6 +4,7 @@ import com.google.protobuf.{ByteString, CodedOutputStream} import scala.jdk.CollectionConverters.* import eu.neverblink.jelly.cli.util.io.YamlDocBuilder import eu.neverblink.jelly.cli.util.io.YamlDocBuilder.* +import eu.neverblink.jelly.core.JellyConstants import eu.neverblink.jelly.core.proto.v1.* import eu.neverblink.protoc.java.runtime.ProtoMessage @@ -26,6 +27,14 @@ object FrameInfo: override def measure(r: String): Long = CodedOutputStream.computeStringSizeNoTag(r) override def name(): String = "size" + /** Measures packed lookup entries: counts their values, or sums the size of the entries. */ + def measureLookup(entries: java.lang.Iterable[RdfLookupEntryPacked])(using + statCollector: StatisticCollector, + ): Long = + if statCollector == CountStatistic then + entries.asScala.iterator.map(_.getValues.size.toLong).sum + else entries.asScala.iterator.map(statCollector.measure).sum + /** This class is used to store the metrics for a single frame */ class FrameInfo(val frameIndex: Long, val metadata: Map[String, ByteString])(using @@ -228,31 +237,57 @@ type Formatter = FrameInfo => Seq[(String, YamlValue)] type DetailFormatter = FrameDetailInfo => Seq[(String, YamlValue)] object MetricsPrinter: - private def withFallback(formatter: DetailFormatter): Formatter = { + /** @param formatter + * formats the details of a Jelly-RDF 1.1 frame + * @param columnFormatter + * formats the details of a Jelly-RDF 1.2 frame + */ + private def withFallback( + formatter: DetailFormatter, + columnFormatter: ColumnFrameInfo => Seq[(String, YamlValue)], + ): Formatter = { + case frame @ (columnInfo: ColumnFrameInfo) => + frame.format().map(_ -> YamlLong(_)) ++ + (if columnInfo.detail then columnFormatter(columnInfo) else Seq()) case frame @ (detailInfo: FrameDetailInfo) => frame.format().map(_ -> YamlLong(_)) ++ formatter(detailInfo) case frame @ (frameInfo: FrameInfo) => frame.format().map(_ -> YamlLong(_)) } - val allFormatter: Formatter = withFallback(detailInfo => { - val splitToTriples = detailInfo.formatAll().map((k, v) => { - val split = k.split("_", 2) - (split(0), split(1), v) - }) - val groupedByTerm = - splitToTriples.groupMap((term, _, _) => term)((_, node, value) => (node, YamlLong(value))) - val mapOfYamlMaps = groupedByTerm.map(_ -> YamlMap(_*)) - val order = IndexedSeq("subject", "predicate", "object", "graph") - mapOfYamlMaps.toSeq.sorted(using Ordering.by[(String, Any), Int](x => order.indexOf(x._1))) - }) - - val termGroupFormatter: Formatter = withFallback(detailInfo => { - Seq("term_details" -> YamlMap(detailInfo.formatGroupByTerm().map(_ -> YamlLong(_))*)) - }) - - val nodeGroupFormatter: Formatter = withFallback(detailInfo => { - Seq("node_details" -> YamlMap(detailInfo.formatGroupByNode().map(_ -> YamlLong(_))*)) - }) + private def yamlLongs(stats: Seq[(String, Long)]): YamlMap = YamlMap(stats.map(_ -> YamlLong(_))*) + + val allFormatter: Formatter = withFallback( + detailInfo => { + val splitToTriples = detailInfo.formatAll().map((k, v) => { + val split = k.split("_", 2) + (split(0), split(1), v) + }) + val groupedByTerm = + splitToTriples.groupMap((term, _, _) => term)((_, node, value) => (node, YamlLong(value))) + val mapOfYamlMaps = groupedByTerm.map(_ -> YamlMap(_*)) + val order = IndexedSeq("subject", "predicate", "object", "graph") + mapOfYamlMaps.toSeq.sorted(using Ordering.by[(String, Any), Int](x => order.indexOf(x._1))) + }, + columnInfo => + columnInfo.valuesByColumn + .map((column, info) => column -> info.format().filter(_._2 > 0)) + .filter(_._2.nonEmpty) + .map((column, stats) => column -> yamlLongs(stats)), + ) + + val termGroupFormatter: Formatter = withFallback( + detailInfo => { + Seq("term_details" -> YamlMap(detailInfo.formatGroupByTerm().map(_ -> YamlLong(_))*)) + }, + columnInfo => Seq("term_details" -> yamlLongs(columnInfo.formatTotalsByColumn())), + ) + + val nodeGroupFormatter: Formatter = withFallback( + detailInfo => { + Seq("node_details" -> YamlMap(detailInfo.formatGroupByNode().map(_ -> YamlLong(_))*)) + }, + columnInfo => Seq("node_details" -> yamlLongs(columnInfo.valuesByKind.format())), + ) class MetricsPrinter(val formatter: Formatter): def printPerFrame( @@ -306,20 +341,33 @@ class MetricsPrinter(val formatter: Formatter): o.write(fullString.getBytes) o.write(System.lineSeparator().getBytes) + /** The stream options, with the fields of the stream's version of Jelly-RDF. */ private def formatOptions( options: RdfStreamOptions, ): YamlMap = - YamlMap( - "stream_name" -> YamlString(options.getStreamName), - "physical_type" -> YamlEnum(options.getPhysicalType.toString, options.getPhysicalTypeValue), - "generalized_statements" -> YamlBool(options.getGeneralizedStatements), - "triple_terms" -> YamlBool(options.getRdfStar), + val tables = Seq( "max_name_table_size" -> YamlInt(options.getMaxNameTableSize), "max_prefix_table_size" -> YamlInt(options.getMaxPrefixTableSize), "max_datatype_table_size" -> YamlInt(options.getMaxDatatypeTableSize), - "logical_type" -> YamlEnum(options.getLogicalType.toString, options.getLogicalTypeValue), - "version" -> YamlInt(options.getVersion), ) + val fields = + if JellyConstants.isRowLayout(options.getVersion) then + Seq( + "stream_name" -> YamlString(options.getStreamName), + "physical_type" -> YamlEnum(options.getPhysicalType, options.getPhysicalTypeValue), + "generalized_statements" -> YamlBool(options.getGeneralizedStatements), + "triple_terms" -> YamlBool(options.getRdfStar), + ) ++ tables ++ Seq( + "logical_type" -> YamlEnum(options.getLogicalType, options.getLogicalTypeValue), + ) + else + Seq( + "stream_name" -> YamlString(options.getStreamName), + "physical_type" -> YamlEnum(options.getPhysicalType, options.getPhysicalTypeValue), + "stream_type" -> YamlEnum(options.getStreamType, options.getStreamTypeValue), + "rdf_version" -> YamlEnum(options.getRdfVersion, options.getRdfVersionValue), + ) ++ tables + YamlMap(fields ++ Seq("version" -> YamlInt(options.getVersion))*) private def formatStatsIndex( frame: FrameInfo, diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/RdfJellySerializationOptions.scala b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/RdfJellySerializationOptions.scala index 10d5a22..f3b4f7a 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/RdfJellySerializationOptions.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/RdfJellySerializationOptions.scala @@ -2,9 +2,12 @@ package eu.neverblink.jelly.cli.command.rdf.util import caseapp.* import eu.neverblink.jelly.cli.InvalidArgument -import eu.neverblink.jelly.core.proto.v1.{LogicalStreamType, PhysicalStreamType, RdfStreamOptions} +import eu.neverblink.jelly.cli.util.args.StreamOptionArgs +import eu.neverblink.jelly.convert.jena.riot.JellyFormatVariant +import eu.neverblink.jelly.core.proto.v1.* import eu.neverblink.jelly.core.utils.LogicalStreamTypeUtils -import eu.neverblink.jelly.core.JellyOptions +import eu.neverblink.jelly.core.{JellyConstants, JellyOptions} +import eu.neverblink.jelly.core.internal.BaseJellyOptions private val `default.opt.streamName`: String = "" private val `default.opt.tripleTerms`: Boolean = true @@ -14,29 +17,38 @@ private val `default.opt.maxDatatypeTableSize`: Int = JellyOptions.BIG_STRICT.getMaxDatatypeTableSize object RdfJellySerializationOptions: - val defaultRowsPerFrame: Int = 256 + def defaultRowsPerFrame(rowLayout: Boolean): Int = + if rowLayout then 256 else JellyFormatVariant.DEFAULT_FRAME_SIZE - /** The stream options rdf to-jelly would use for input in this format, with no options set. */ - def defaultFor(inputFormat: RdfFormat): RdfStreamOptions = - val options = RdfJellySerializationOptions() - options.inferGeneralized(Some(inputFormat.cliOptions.head), None) - options.asRdfStreamOptions + def defaultOptions: RdfStreamOptions = RdfJellySerializationOptions().asRdfStreamOptions /** Options for serializing in Jelly-RDF */ case class RdfJellySerializationOptions( + @HelpMessage( + "Version of Jelly-RDF to write. One of: 1.2, 1.1. Jelly-RDF 1.1 is deprecated, use it " + + "only for readers that do not support 1.2. Default: 1.2", + ) + `opt.jellyVersion`: Option[String] = None, @HelpMessage("Name of the output stream (in metadata). Default: (empty)") `opt.streamName`: Option[String] = None, @HelpMessage( - "Whether the stream may contain generalized triples, quads, or datasets. Default: (true for N-Triples/N-Quads and Jena binary formats, false otherwise)", + "Physical stream type. One of: TRIPLES, QUADS, GRAPHS (only Jelly-RDF 1.1). " + + "Default: either TRIPLES or QUADS, depending on the input format.", ) - `opt.generalizedStatements`: Option[Boolean] = None, + `opt.physicalType`: Option[String] = None, @HelpMessage( - "Whether the stream may contain RDF 1.2 triple terms. Default: " + - `default.opt.tripleTerms`, + "Only Jelly-RDF 1.2. Type of the stream. One of: flat (one RDF graph or dataset), " + + "messages (a stream of RDF Messages). Default: flat", ) - `opt.tripleTerms`: Option[Boolean] = None, + `opt.streamType`: Option[String] = None, + @HelpMessage( + "Only Jelly-RDF 1.2. " + StreamOptionArgs.rdfVersionHelp + " Default: unspecified", + ) + `opt.rdfVersion`: Option[String] = None, @HelpMessage( - "Maximum size of the name lookup table. Default: " + `default.opt.maxNameTableSize`, + "Maximum size of the name lookup table. Must be at least " + + BaseJellyOptions.MIN_COLUMN_NAME_TABLE_SIZE + " in Jelly-RDF 1.2. Default: " + + `default.opt.maxNameTableSize`, ) `opt.maxNameTableSize`: Option[Int] = None, @HelpMessage( @@ -48,13 +60,18 @@ case class RdfJellySerializationOptions( ) `opt.maxDatatypeTableSize`: Option[Int] = None, @HelpMessage( - "Physical stream type. One of: TRIPLES, QUADS, GRAPHS. " + - "Default: either TRIPLES or QUADS, depending on the input format.", + "Only Jelly-RDF 1.1. Whether the stream may contain generalized triples, quads, or " + + "datasets. Default: (true for N-Triples/N-Quads and Jena binary formats, false otherwise)", ) - `opt.physicalType`: Option[String] = None, + `opt.generalizedStatements`: Option[Boolean] = None, + @HelpMessage( + "Only Jelly-RDF 1.1. Whether the stream may contain RDF 1.2 triple terms. Default: " + + `default.opt.tripleTerms`, + ) + `opt.tripleTerms`: Option[Boolean] = None, @HelpMessage( - "Logical (RDF-STaX-based) stream type. This can be either a name like " + - "`FLAT_QUADS` or a full IRI like `https://w3id.org/stax/ontology#flatQuadStream`. " + + "Only Jelly-RDF 1.1. Logical (RDF-STaX-based) stream type. This can be either a name " + + "like `FLAT_QUADS` or a full IRI like `https://w3id.org/stax/ontology#flatQuadStream`. " + "Default: (unspecified)", ) `opt.logicalType`: Option[String] = None, @@ -76,6 +93,15 @@ case class RdfJellySerializationOptions( case _ => false } + /** Whether the output is Jelly-RDF 1.0 or 1.1 (row layout), rather than Jelly-RDF 1.2. */ + lazy val isRowLayout: Boolean = + `opt.jellyVersion`.map(_.trim) match + case Some("1.1") => true + case Some("1.2") => false + case Some(x) => + throw InvalidArgument("--opt.jelly-version", x, Some("Must be one of: 1.2, 1.1")) + case None => inferred.options.exists(StreamOptionsUtil.isRowLayout) + private lazy val logicalType: Option[LogicalStreamType] = val logicalIri = `opt.logicalType` .map(_.trim).filter(_.nonEmpty) @@ -111,40 +137,107 @@ case class RdfJellySerializationOptions( ) case None => PhysicalStreamType.UNSPECIFIED - private def makeStreamOptions(): RdfStreamOptions = - RdfStreamOptions.newInstance() - .setStreamName(`opt.streamName`.getOrElse(`default.opt.streamName`)) - .setGeneralizedStatements(`opt.generalizedStatements`.getOrElse(inferred.generalized)) - .setRdfStar(`opt.tripleTerms`.getOrElse(`default.opt.tripleTerms`)) - .setMaxNameTableSize(`opt.maxNameTableSize`.getOrElse(`default.opt.maxNameTableSize`)) - .setMaxPrefixTableSize(`opt.maxPrefixTableSize`.getOrElse(`default.opt.maxPrefixTableSize`)) - .setMaxDatatypeTableSize( - `opt.maxDatatypeTableSize`.getOrElse(`default.opt.maxDatatypeTableSize`), + private lazy val streamType: Option[RdfStreamType] = + `opt.streamType`.map(_.trim.toLowerCase).map { + case "flat" => RdfStreamType.FLAT + case "messages" => RdfStreamType.MESSAGES + case _ => + throw InvalidArgument( + "--opt.stream-type", + `opt.streamType`.get, + Some("Must be one of: flat, messages"), + ) + } + + /** Throws if an option of one version of Jelly-RDF is set while writing the other one. */ + private def checkVersionSpecificOptions(): Unit = + def onlyIn(version: String, argument: String, value: Option[Any], hint: String): Unit = + value.foreach { v => + throw InvalidArgument( + argument, + v.toString, + Some(s"This option is only valid in Jelly-RDF $version output. $hint"), + ) + } + if isRowLayout then + val hint = "Use --opt.jelly-version=1.2 to write Jelly-RDF 1.2." + onlyIn("1.2", "--opt.stream-type", `opt.streamType`, hint) + onlyIn("1.2", "--opt.rdf-version", `opt.rdfVersion`, hint) + else + val hint = "Use --opt.jelly-version=1.1 to write Jelly-RDF 1.1." + onlyIn( + "1.1", + "--opt.generalized-statements", + `opt.generalizedStatements`, + "Jelly-RDF 1.2 cannot store generalized statements. " + hint, + ) + onlyIn( + "1.1", + "--opt.triple-terms", + `opt.tripleTerms`, + "In Jelly-RDF 1.2, use --opt.rdf-version instead. " + hint, + ) + onlyIn( + "1.1", + "--opt.logical-type", + `opt.logicalType`, + "In Jelly-RDF 1.2, use --opt.stream-type instead. " + hint, + ) + onlyIn( + "1.1", + "--opt.physical-type", + `opt.physicalType`.filter(_ => physicalType == PhysicalStreamType.GRAPHS), + "In Jelly-RDF 1.2, use QUADS instead. " + hint, + ) + StreamOptionArgs.checkTableSize( + "--opt.max-name-table-size", + `opt.maxNameTableSize`, + BaseJellyOptions.MIN_COLUMN_NAME_TABLE_SIZE, ) - .setPhysicalType(physicalType) - .setLogicalType(logicalType.getOrElse(LogicalStreamType.UNSPECIFIED)) - private lazy val optionsFromFileWithOverrides: Option[RdfStreamOptions] = - inferred.options.map(x => { - val cloned = x.clone() - if `opt.generalizedStatements`.isDefined then - cloned.setGeneralizedStatements(`opt.generalizedStatements`.get) - if `opt.streamName`.isDefined then // comment to stop scalafmt from making this a mess - cloned.setStreamName(`opt.streamName`.get) - if `opt.tripleTerms`.isDefined then // comment to stop scalafmt from making this a mess - cloned.setRdfStar(`opt.tripleTerms`.get) - if `opt.maxNameTableSize`.isDefined then - cloned.setMaxNameTableSize(`opt.maxNameTableSize`.get) - if `opt.maxPrefixTableSize`.isDefined then - cloned.setMaxPrefixTableSize(`opt.maxPrefixTableSize`.get) - if `opt.maxDatatypeTableSize`.isDefined then - cloned.setMaxDatatypeTableSize(`opt.maxDatatypeTableSize`.get) - if `opt.physicalType`.isDefined then // comment to stop scalafmt from making this a mess - cloned.setPhysicalType(physicalType) - if `opt.logicalType`.isDefined then - cloned.setLogicalType(logicalType.getOrElse(LogicalStreamType.UNSPECIFIED)) - cloned - }) + /** The options to start from, before the ones set on the command line: those of the + * --options-from file, made fit for the output version, or the defaults. + */ + private def baseOptions: RdfStreamOptions.Mutable = + val defaults = RdfStreamOptions.newInstance() + .setStreamName(`default.opt.streamName`) + .setMaxNameTableSize(`default.opt.maxNameTableSize`) + .setMaxPrefixTableSize(`default.opt.maxPrefixTableSize`) + .setMaxDatatypeTableSize(`default.opt.maxDatatypeTableSize`) + val rowLayoutDefaults = (o: RdfStreamOptions.Mutable) => + o.setGeneralizedStatements(inferred.generalized).setRdfStar(`default.opt.tripleTerms`) + inferred.options match + case None => if isRowLayout then rowLayoutDefaults(defaults) else defaults + case Some(fromFile) if StreamOptionsUtil.isRowLayout(fromFile) == isRowLayout => + fromFile.clone() + case Some(fromFile) => + // The file is of the other version of Jelly-RDF, so only take the options both have + defaults + .setStreamName(fromFile.getStreamName) + .setPhysicalType(fromFile.getPhysicalType match + case PhysicalStreamType.GRAPHS => PhysicalStreamType.QUADS + case t => t) + .setMaxNameTableSize(fromFile.getMaxNameTableSize) + .setMaxPrefixTableSize(fromFile.getMaxPrefixTableSize) + .setMaxDatatypeTableSize(fromFile.getMaxDatatypeTableSize) + if isRowLayout then rowLayoutDefaults(defaults) else defaults lazy val asRdfStreamOptions: RdfStreamOptions = - optionsFromFileWithOverrides.getOrElse(makeStreamOptions()) + checkVersionSpecificOptions() + val options = baseOptions + `opt.streamName`.foreach(options.setStreamName) + if `opt.physicalType`.isDefined then options.setPhysicalType(physicalType) + `opt.maxNameTableSize`.foreach(options.setMaxNameTableSize) + `opt.maxPrefixTableSize`.foreach(options.setMaxPrefixTableSize) + `opt.maxDatatypeTableSize`.foreach(options.setMaxDatatypeTableSize) + if isRowLayout then + `opt.generalizedStatements`.foreach(options.setGeneralizedStatements) + `opt.tripleTerms`.foreach(options.setRdfStar) + if `opt.logicalType`.isDefined then + options.setLogicalType(logicalType.getOrElse(LogicalStreamType.UNSPECIFIED)) + // The writer sets the exact version (1.0 or 1.1), depending on the namespace declarations + options.setVersion(JellyConstants.PROTO_VERSION_1_1_X) + else + streamType.foreach(options.setStreamType) + `opt.rdfVersion`.map(StreamOptionArgs.rdfVersion).foreach(options.setRdfVersion) + options.setVersion(JellyConstants.PROTO_VERSION_1_2_X) diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/StreamOptionsUtil.scala b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/StreamOptionsUtil.scala index 57471c9..542730f 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/StreamOptionsUtil.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/StreamOptionsUtil.scala @@ -1,9 +1,40 @@ package eu.neverblink.jelly.cli.command.rdf.util import eu.neverblink.jelly.cli.util.io.ProtoText -import eu.neverblink.jelly.core.proto.v1.RdfStreamOptions +import eu.neverblink.jelly.core.JellyConstants +import eu.neverblink.jelly.core.proto.v1.{RdfStreamOptions, RdfVersion} object StreamOptionsUtil: def prettyPrint(streamOptions: RdfStreamOptions): String = ProtoText.print(RdfStreamOptions.getDescriptor, streamOptions.toByteArray) + + /** Whether the options announce a Jelly-RDF 1.0 or 1.1 stream (row layout). Version 0 (not set) + * means Jelly-RDF 1.2. + */ + def isRowLayout(options: RdfStreamOptions): Boolean = + JellyConstants.requestsRowLayout(options.getVersion) + + /** Whether a stream with these options may contain generalized statements. */ + def allowsGeneralized(options: RdfStreamOptions): Boolean = + isRowLayout(options) && options.getGeneralizedStatements + + /** Whether a stream with these options may contain triple terms. */ + def allowsTripleTerms(options: RdfStreamOptions): Boolean = + if isRowLayout(options) then options.getRdfStar + else + options.getRdfVersion == RdfVersion.RDF_VERSION_UNSPECIFIED || + options.getRdfVersion == RdfVersion.RDF_VERSION_1_2 + + /** How to allow generalized statements in a stream with these options, for error messages. */ + def generalizedHint(options: RdfStreamOptions): String = + if isRowLayout(options) then + "Use --opt.generalized-statements=true to allow generalized statements." + else + "Jelly-RDF 1.2 cannot store generalized statements. Use --opt.jelly-version=1.1 " + + "--opt.generalized-statements=true to write them in Jelly-RDF 1.1." + + /** How to allow triple terms in a stream with these options, for error messages. */ + def tripleTermsHint(options: RdfStreamOptions): String = + if isRowLayout(options) then "Use --opt.triple-terms=true to allow triple terms." + else "Use --opt.rdf-version=1.2 to allow triple terms." diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlFromJelly.scala b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlFromJelly.scala index 4d448b4..da8dbef 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlFromJelly.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlFromJelly.scala @@ -2,7 +2,7 @@ package eu.neverblink.jelly.cli.command.sparql import caseapp.* import eu.neverblink.jelly.cli.* -import eu.neverblink.jelly.cli.command.rdf.util.{RdfFormat, RdfJellySerializationOptions} +import eu.neverblink.jelly.cli.command.rdf.util.RdfJellySerializationOptions import eu.neverblink.jelly.cli.command.sparql.util.SparqlFormat import eu.neverblink.jelly.cli.util.jena.RdfSparqlConverter @@ -67,7 +67,8 @@ object SparqlFromJelly extends SparqlSerDesCommand[SparqlFromJellyOptions]: RdfSparqlConverter.sparqlToRdf( inputStream, outputStream, - RdfJellySerializationOptions.defaultFor(RdfFormat.JellySparql), - RdfJellySerializationOptions.defaultRowsPerFrame, + RdfJellySerializationOptions.defaultOptions, + streamTypeSet = false, + RdfJellySerializationOptions.defaultRowsPerFrame(rowLayout = false), delimited = true, ) diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlJellySerializationOptions.scala b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlJellySerializationOptions.scala index 6be04d3..9a7f6f2 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlJellySerializationOptions.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlJellySerializationOptions.scala @@ -2,10 +2,9 @@ package eu.neverblink.jelly.cli.command.sparql.util import caseapp.* import eu.neverblink.jelly.cli.InvalidArgument -import eu.neverblink.jelly.core.proto.v1.RdfVersion +import eu.neverblink.jelly.cli.util.args.StreamOptionArgs import eu.neverblink.jelly.core.proto.v1.sparql.{SparqlResultsOptions, SparqlStreamType} import eu.neverblink.jelly.core.sparql.JellySparqlOptions -import eu.neverblink.jelly.core.utils.RdfVersionUtils private val defaultOptions: SparqlResultsOptions = JellySparqlOptions.BIG @@ -22,11 +21,7 @@ case class SparqlJellySerializationOptions( "sets). Default: flat, or punctuated if there is more than one input file", ) `opt.streamType`: Option[String] = None, - @HelpMessage( - "Version of RDF whose terms may occur in the stream. One of: 1.1 (no triple terms, " + - "no base directions), 1.2-basic (base directions, no triple terms), 1.2 (all RDF 1.2 " + - "terms), unspecified. Default: unspecified", - ) + @HelpMessage(StreamOptionArgs.rdfVersionHelp + " Default: unspecified") `opt.rdfVersion`: Option[String] = None, @HelpMessage( "Maximum size of the name lookup table. Must be at least " + @@ -57,25 +52,6 @@ case class SparqlJellySerializationOptions( ) } - private def rdfVersion: Option[RdfVersion] = - `opt.rdfVersion`.map(_.trim.toLowerCase).map { - case "unspecified" => RdfVersion.RDF_VERSION_UNSPECIFIED - case label => - try RdfVersionUtils.rdfVersionFromLabel(label) - catch - case _: IllegalArgumentException => - throw InvalidArgument( - "--opt.rdf-version", - `opt.rdfVersion`.get, - Some("Must be one of: 1.1, 1.2-basic, 1.2, unspecified"), - ) - } - - private def checkTableSize(argument: String, size: Option[Int], minSize: Int): Unit = - size.filter(_ < minSize).foreach { s => - throw InvalidArgument(argument, s.toString, Some(f"Must be at least $minSize")) - } - /** Builds the stream options, starting from `base` (or the default preset) and applying the * options set on the command line on top. * @@ -85,17 +61,21 @@ case class SparqlJellySerializationOptions( * if any of the options is invalid */ def toSparqlResultsOptions(base: Option[SparqlResultsOptions] = None): SparqlResultsOptions = - checkTableSize( + StreamOptionArgs.checkTableSize( "--opt.max-name-table-size", `opt.maxNameTableSize`, JellySparqlOptions.MIN_NAME_TABLE_SIZE, ) - checkTableSize("--opt.max-prefix-table-size", `opt.maxPrefixTableSize`, 0) - checkTableSize("--opt.max-datatype-table-size", `opt.maxDatatypeTableSize`, 0) + StreamOptionArgs.checkTableSize("--opt.max-prefix-table-size", `opt.maxPrefixTableSize`, 0) + StreamOptionArgs.checkTableSize( + "--opt.max-datatype-table-size", + `opt.maxDatatypeTableSize`, + 0, + ) val options = base.getOrElse(defaultOptions).clone() `opt.streamName`.foreach(options.setStreamName) streamType.foreach(options.setStreamType) - rdfVersion.foreach(options.setRdfVersion) + `opt.rdfVersion`.map(StreamOptionArgs.rdfVersion).foreach(options.setRdfVersion) `opt.maxNameTableSize`.foreach(options.setMaxNameTableSize) `opt.maxPrefixTableSize`.foreach(options.setMaxPrefixTableSize) `opt.maxDatatypeTableSize`.foreach(options.setMaxDatatypeTableSize) diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlJellyUtil.scala b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlJellyUtil.scala index 31cef5b..8d5956f 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlJellyUtil.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlJellyUtil.scala @@ -5,7 +5,8 @@ import eu.neverblink.jelly.cli.* import eu.neverblink.jelly.cli.util.io.IoUtil import eu.neverblink.jelly.core.{RdfProtoDeserializationError, RdfProtoSerializationError} import eu.neverblink.jelly.core.proto.v1.sparql.{SparqlResultsFrame, SparqlResultsOptions} -import eu.neverblink.jelly.core.sparql.{JellySparqlIoUtils, JellySparqlOptions} +import eu.neverblink.jelly.core.sparql.JellySparqlOptions +import eu.neverblink.jelly.core.utils.IoUtils import org.apache.jena.riot.RiotException import java.io.InputStream @@ -25,7 +26,7 @@ object SparqlJellyUtil: def iterateSparqlStreamWithDelimitingInfo( inputStream: InputStream, ): (Boolean, Iterator[SparqlResultsFrame]) = - val response = JellySparqlIoUtils.autodetectDelimiting(inputStream) + val response = IoUtils.autodetectDelimiting(inputStream, SparqlResultsFrame.getDescriptor) val input = response.newInput() if response.isDelimited then ( diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlMetricsPrinter.scala b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlMetricsPrinter.scala index 575fabf..228df7a 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlMetricsPrinter.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlMetricsPrinter.scala @@ -1,20 +1,18 @@ package eu.neverblink.jelly.cli.command.sparql.util import com.google.protobuf.ByteString -import eu.neverblink.jelly.cli.command.rdf.util.FrameInfo +import eu.neverblink.jelly.cli.command.rdf.util.{ColumnInfo, FrameInfo} import eu.neverblink.jelly.cli.util.io.YamlDocBuilder import eu.neverblink.jelly.cli.util.io.YamlDocBuilder.* -import eu.neverblink.jelly.core.proto.v1.{RdfColumn, RdfLookupEntryPacked} import eu.neverblink.jelly.core.proto.v1.sparql.* -import eu.neverblink.protoc.java.runtime.ProtoMessage import java.io.OutputStream import scala.jdk.CollectionConverters.* /** Statistics of a single frame of a Jelly-SPARQL stream, or of several frames added together. * - * With the count statistic, lookup entries and column values are counted one by one (a packed - * lookup entry holds many values). With the size statistic, their serialized size is summed. + * See [[eu.neverblink.jelly.cli.command.rdf.util.ColumnInfo]] for how the lookup entries and + * column values are measured. */ final class SparqlFrameInfo( val frameIndex: Long, @@ -40,11 +38,7 @@ final class SparqlFrameInfo( var name: Long = 0 var prefix: Long = 0 var datatype: Long = 0 - var iri: Long = 0 - var bnode: Long = 0 - var literal: Long = 0 - var tripleTerm: Long = 0 - var layout: Long = 0 + val columns: ColumnInfo = ColumnInfo() var askResult: Long = 0 var trailer: Long = 0 @@ -59,28 +53,11 @@ final class SparqlFrameInfo( this.stat.name += other.stat.name this.stat.prefix += other.stat.prefix this.stat.datatype += other.stat.datatype - this.stat.iri += other.stat.iri - this.stat.bnode += other.stat.bnode - this.stat.literal += other.stat.literal - this.stat.tripleTerm += other.stat.tripleTerm - this.stat.layout += other.stat.layout + this.stat.columns += other.stat.columns this.stat.askResult += other.stat.askResult this.stat.trailer += other.stat.trailer this - private def isCount: Boolean = statCollector == FrameInfo.CountStatistic - - private def measureAll(messages: Iterable[ProtoMessage[?]]): Long = - messages.iterator.map(statCollector.measure).sum - - private def measureLookup(entries: Iterable[RdfLookupEntryPacked]): Long = - if isCount then entries.iterator.map(_.getValues.size.toLong).sum - else measureAll(entries) - - /** The number of values of one kind in a column, or the size of the fields holding them. */ - private def measureValues(count: Int, size: => Long): Long = - if isCount then count else size - /** @param resultSetStart * whether this is the first frame of a result set */ @@ -92,33 +69,16 @@ final class SparqlFrameInfo( stat.row += frame.getRowCount Option(frame.getOptions).foreach(o => stat.option += statCollector.measure(o)) stat.variable += frame.getVariables.asScala.iterator.map(statCollector.measure).sum - stat.name += measureLookup(frame.getNames.asScala) - stat.prefix += measureLookup(frame.getPrefixes.asScala) - stat.datatype += measureLookup(frame.getDatatypes.asScala) - frame.getColumns.asScala.foreach(processColumn) + stat.name += FrameInfo.measureLookup(frame.getNames) + stat.prefix += FrameInfo.measureLookup(frame.getPrefixes) + stat.datatype += FrameInfo.measureLookup(frame.getDatatypes) + frame.getColumns.asScala.foreach(stat.columns.process) Option(frame.getAskResult).foreach(r => stat.askResult += statCollector.measure(r)) Option(frame.getTrailer).foreach { t => stat.trailer += statCollector.measure(t) if t.getError.nonEmpty then trailerError = Some(t.getError) } - private def processColumn(c: RdfColumn): Unit = - stat.iri += measureValues( - c.getNameIds.size, - c.getNameIds.computeUInt32SizeNoTag + c.getPrefixIds.computeUInt32SizeNoTag, - ) - stat.literal += measureValues( - c.getLexValues.size, - c.getLexValues.computeStringSizeNoTag + c.getLiteralKinds.computeUInt32SizeNoTag + - c.getLangtags.computeStringSizeNoTag + c.getLangtagDirections.computeUInt32SizeNoTag, - ) - stat.bnode += measureValues(c.getBnodes.size, c.getBnodes.computeStringSizeNoTag) - stat.tripleTerm += measureValues(c.getTripleTerms.size, measureAll(c.getTripleTerms.asScala)) - stat.layout += measureValues( - c.getLayouts.size, - c.getLayouts.computeUInt32SizeNoTag + c.getKinds.size, - ) - def format(): Seq[(String, YamlValue)] = val name = statCollector.name() Seq( @@ -130,11 +90,7 @@ final class SparqlFrameInfo( "name_" + name -> YamlLong(stat.name), "prefix_" + name -> YamlLong(stat.prefix), "datatype_" + name -> YamlLong(stat.datatype), - "iri_value_" + name -> YamlLong(stat.iri), - "bnode_value_" + name -> YamlLong(stat.bnode), - "literal_value_" + name -> YamlLong(stat.literal), - "triple_term_value_" + name -> YamlLong(stat.tripleTerm), - "layout_" + name -> YamlLong(stat.layout), + ) ++ stat.columns.format().map(_ -> YamlLong(_)) ++ Seq( "ask_result_" + name -> YamlLong(stat.askResult), "trailer_" + name -> YamlLong(stat.trailer), ) ++ trailerError.map(e => "trailer_error" -> YamlString(e)) @@ -168,14 +124,8 @@ object SparqlMetricsPrinter: val header = Seq( "stream_options" -> YamlMap( "stream_name" -> YamlString(options.getStreamName), - "stream_type" -> YamlEnum( - Option(options.getStreamType).map(_.toString).getOrElse("UNKNOWN"), - options.getStreamTypeValue, - ), - "rdf_version" -> YamlEnum( - Option(options.getRdfVersion).map(_.toString).getOrElse("UNKNOWN"), - options.getRdfVersionValue, - ), + "stream_type" -> YamlEnum(options.getStreamType, options.getStreamTypeValue), + "rdf_version" -> YamlEnum(options.getRdfVersion, options.getRdfVersionValue), "max_name_table_size" -> YamlInt(options.getMaxNameTableSize), "max_prefix_table_size" -> YamlInt(options.getMaxPrefixTableSize), "max_datatype_table_size" -> YamlInt(options.getMaxDatatypeTableSize), diff --git a/src/main/scala/eu/neverblink/jelly/cli/util/args/StreamOptionArgs.scala b/src/main/scala/eu/neverblink/jelly/cli/util/args/StreamOptionArgs.scala new file mode 100644 index 0000000..afd59ce --- /dev/null +++ b/src/main/scala/eu/neverblink/jelly/cli/util/args/StreamOptionArgs.scala @@ -0,0 +1,31 @@ +package eu.neverblink.jelly.cli.util.args + +import eu.neverblink.jelly.cli.InvalidArgument +import eu.neverblink.jelly.core.proto.v1.RdfVersion +import eu.neverblink.jelly.core.utils.RdfVersionUtils + +/** Parsing of the --opt.* arguments shared by Jelly-RDF and Jelly-SPARQL. */ +object StreamOptionArgs: + val rdfVersionHelp: String = + "Version of RDF for the stream. One of: 1.1, 1.2-basic (base directions, no triple terms), " + + "1.2 (all RDF 1.2 terms), unspecified." + + /** Parses the value of --opt.rdf-version. */ + def rdfVersion(value: String): RdfVersion = + value.trim.toLowerCase match + case "unspecified" => RdfVersion.RDF_VERSION_UNSPECIFIED + case label => + try RdfVersionUtils.rdfVersionFromLabel(label) + catch + case _: IllegalArgumentException => + throw InvalidArgument( + "--opt.rdf-version", + value, + Some("Must be one of: 1.1, 1.2-basic, 1.2, unspecified"), + ) + + /** Checks that a lookup table size set on the command line is at least `minSize`. */ + def checkTableSize(argument: String, size: Option[Int], minSize: Int): Unit = + size.filter(_ < minSize).foreach { s => + throw InvalidArgument(argument, s.toString, Some(f"Must be at least $minSize")) + } diff --git a/src/main/scala/eu/neverblink/jelly/cli/util/io/YamlDocBuilder.scala b/src/main/scala/eu/neverblink/jelly/cli/util/io/YamlDocBuilder.scala index bcb5dcf..4f50710 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/util/io/YamlDocBuilder.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/util/io/YamlDocBuilder.scala @@ -56,6 +56,11 @@ object YamlDocBuilder: sealed trait YamlScalar extends YamlValue case class YamlBlank() extends YamlScalar case class YamlEnum(v: String, i: Int) extends YamlScalar + + object YamlEnum: + /** A protobuf enum value. */ + def apply(value: AnyRef, number: Int): YamlEnum = + YamlEnum(Option(value).map(_.toString).getOrElse("UNKNOWN"), number) case class YamlInt(v: Int) extends YamlScalar case class YamlLong(v: Long) extends YamlScalar case class YamlBool(v: Boolean) extends YamlScalar diff --git a/src/main/scala/eu/neverblink/jelly/cli/util/jena/RdfSparqlConverter.scala b/src/main/scala/eu/neverblink/jelly/cli/util/jena/RdfSparqlConverter.scala index c35ca52..19392f8 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/util/jena/RdfSparqlConverter.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/util/jena/RdfSparqlConverter.scala @@ -1,13 +1,17 @@ package eu.neverblink.jelly.cli.util.jena import eu.neverblink.jelly.cli.* -import eu.neverblink.jelly.cli.command.rdf.util.JellyUtil +import eu.neverblink.jelly.cli.command.rdf.util.{JellyUtil, StreamOptionsUtil} import eu.neverblink.jelly.cli.command.sparql.util.{ SparqlJellyUtil, SparqlResultSet, SparqlResultSetReader, } -import eu.neverblink.jelly.cli.util.jena.riot.{JellyFrameWriter, JellyWriterUtil} +import eu.neverblink.jelly.cli.util.jena.riot.{ + JellyFrameWriter, + JellyMessageWriter, + JellyWriterUtil, +} import eu.neverblink.jelly.convert.jena.JenaConverterFactory import eu.neverblink.jelly.convert.jena.sparql.JellySparqlLanguage import eu.neverblink.jelly.core.JellyOptions @@ -41,7 +45,9 @@ object RdfSparqlConverter: */ def defaultSparqlOptions(inputOptions: RdfStreamOptions): SparqlResultsOptions = JellySparqlOptions.BIG.clone().setRdfVersion( - if inputOptions.getRdfStar then RdfVersion.RDF_VERSION_1_2 else RdfVersion.RDF_VERSION_1_1, + if !StreamOptionsUtil.isRowLayout(inputOptions) then inputOptions.getRdfVersion + else if inputOptions.getRdfStar then RdfVersion.RDF_VERSION_1_2 + else RdfVersion.RDF_VERSION_1_1, ) /** Converts a Jelly-RDF stream to a Jelly-SPARQL result set, one solution per statement. @@ -86,10 +92,7 @@ object RdfSparqlConverter: inputStream: InputStream, ): (RdfStreamOptions, Seq[Var], Iterator[Binding]) = val frames = JellyUtil.iterateRdfStream(inputStream).buffered - val inputOptions = frames.headOption.flatMap(_.getRows.asScala.headOption) match - case Some(row) if row.hasOptions => row.getOptions - case Some(_) => throw CriticalException("First input row is not an options row") - case None => throw CriticalException("Empty input stream") + val inputOptions = JellyUtil.requireFirstOptions(frames) val quads = inputOptions.getPhysicalType match case PhysicalStreamType.TRIPLES => false case PhysicalStreamType.QUADS | PhysicalStreamType.GRAPHS => true @@ -118,21 +121,24 @@ object RdfSparqlConverter: val bindings = frames.flatMap { (frame: RdfStreamFrame) => // A new buffer for every frame, so that the iterator can take the old one as it is buffer = ArrayBuffer[Binding]() - frame.getRows.forEach(decoder.ingestRow) + decoder.ingestFrame(frame) buffer } (inputOptions, vars, bindings) /** Converts a Jelly-SPARQL stream to a Jelly-RDF stream, one statement per solution. * - * A FLAT stream gives a stream with frames of `rowsPerFrame` rows. A PUNCTUATED stream gives one - * frame per result set, with the logical type set to GRAPHS (TRIPLES output) or DATASETS - * (otherwise), unless it was set already. + * A FLAT stream is a stream with frames of `rowsPerFrame` rows. A PUNCTUATED stream in Jelly-RDF + * 1.2 is a stream of RDF Messages with one message per result set, unless the stream type was + * set. In Jelly-RDF 1.1, it yields one frame per result set, with the logical type set to GRAPHS + * (TRIPLES output) or DATASETS (otherwise), unless it was set already. * * @param outputOptions * stream options of the output. The physical type may be unspecified. + * @param streamTypeSet + * whether the stream type in `outputOptions` was set by the user, and must be kept * @param rowsPerFrame - * target number of rows per output frame, for FLAT input + * target number of statements (Jelly-RDF 1.2) or rows (Jelly-RDF 1.1) per output frame * @param delimited * whether the output should be delimited */ @@ -140,6 +146,7 @@ object RdfSparqlConverter: inputStream: InputStream, outputStream: OutputStream, outputOptions: RdfStreamOptions, + streamTypeSet: Boolean, rowsPerFrame: Int, delimited: Boolean, ): Unit = @@ -148,7 +155,17 @@ object RdfSparqlConverter: if !resultSets.hasNext then throw CriticalException("Empty input stream") val first = selectResult(resultSets.next(), 0) val jellyOpt = withPhysicalType(outputOptions, first.variables.size == 4) - if resultSets.isPunctuated then + val rowLayout = StreamOptionsUtil.isRowLayout(jellyOpt) + if resultSets.isPunctuated && !rowLayout && !streamTypeSet then + jellyOpt.setStreamType(RdfStreamType.MESSAGES) + if !rowLayout && jellyOpt.getStreamType == RdfStreamType.MESSAGES then + val writer = JellyMessageWriter(jellyOpt, rowsPerFrame, delimited, outputStream) + writeStatements(first, writer, jellyOpt, Option.when(resultSets.isPunctuated)(0)) + for (resultSet, i) <- resultSets.zipWithIndex do + writer.endMessage() + writeStatements(selectResult(resultSet, i + 1), writer, jellyOpt, Some(i + 1)) + writer.finish() + else if resultSets.isPunctuated && rowLayout then if jellyOpt.getLogicalType == LogicalStreamType.UNSPECIFIED then jellyOpt.setLogicalType( if jellyOpt.getPhysicalType == PhysicalStreamType.TRIPLES then LogicalStreamType.GRAPHS @@ -171,6 +188,9 @@ object RdfSparqlConverter: ) writer.start() writeStatements(first, writer, jellyOpt) + // With the FLAT stream type set, all result sets are written as one graph or dataset + for (resultSet, i) <- resultSets.zipWithIndex do + writeStatements(selectResult(resultSet, i + 1), writer, jellyOpt, Some(i + 1)) writer.finish() outputStream.flush() } @@ -210,7 +230,7 @@ object RdfSparqlConverter: if quads && triples then throw CriticalException( s"Result set${resultSetIndex.map(" " + _).getOrElse("")} has 4 variables, but the output is a " + - "TRIPLES stream. Use --opt.physical-type=QUADS or GRAPHS.", + "TRIPLES stream. Use --opt.physical-type=QUADS.", ) var index = 0L for row <- resultSet.rows do @@ -262,7 +282,7 @@ object RdfSparqlConverter: throw InvalidArgument( "--opt.physical-type", "TRIPLES", - Some("The result set has 4 variables, so the output must be QUADS or GRAPHS"), + Some("The result set has 4 variables, so the output must be QUADS"), ) case _ => jellyOpt @@ -273,13 +293,13 @@ object RdfSparqlConverter: solution: String, opt: RdfStreamOptions, ): Unit = - if !opt.getGeneralizedStatements && generalized then + if generalized && !StreamOptionsUtil.allowsGeneralized(opt) then throw CriticalException( s"Solution $solution is not a valid RDF statement: $statement. " + - "Use --opt.generalized-statements=true to allow generalized statements.", + StreamOptionsUtil.generalizedHint(opt), ) - if !opt.getRdfStar && tripleTerms then + if tripleTerms && !StreamOptionsUtil.allowsTripleTerms(opt) then throw CriticalException( s"Solution $solution contains a triple term: $statement. " + - "Use --opt.triple-terms=true to allow triple terms.", + StreamOptionsUtil.tripleTermsHint(opt), ) diff --git a/src/main/scala/eu/neverblink/jelly/cli/util/jena/riot/JellyFrameWriter.scala b/src/main/scala/eu/neverblink/jelly/cli/util/jena/riot/JellyFrameWriter.scala index 618a111..b65fb41 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/util/jena/riot/JellyFrameWriter.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/util/jena/riot/JellyFrameWriter.scala @@ -10,6 +10,7 @@ import org.apache.jena.riot.system.StreamRDF import org.apache.jena.sparql.core.Quad import java.io.OutputStream +import scala.annotation.nowarn /** Writes Jelly-RDF, ending a frame only when [[endFrame]] is called. This is for grouped streams, * where each frame is one graph or dataset. @@ -24,6 +25,8 @@ import java.io.OutputStream final class JellyFrameWriter(options: RdfStreamOptions, delimited: Boolean, out: OutputStream) extends StreamRDF: private val buffer = RowBuffer.newLazyImmutable() + // The Jelly-RDF 1.1 encoder is deprecated, but still needed for --opt.jelly-version=1.1 + @nowarn("cat=deprecation") private val encoder = JenaConverterFactory.getInstance().encoder( ProtoEncoder.Params.of(options, false, buffer), ) diff --git a/src/main/scala/eu/neverblink/jelly/cli/util/jena/riot/JellyMessageWriter.scala b/src/main/scala/eu/neverblink/jelly/cli/util/jena/riot/JellyMessageWriter.scala new file mode 100644 index 0000000..036bfaa --- /dev/null +++ b/src/main/scala/eu/neverblink/jelly/cli/util/jena/riot/JellyMessageWriter.scala @@ -0,0 +1,68 @@ +package eu.neverblink.jelly.cli.util.jena.riot + +import eu.neverblink.jelly.cli.CriticalException +import eu.neverblink.jelly.convert.jena.JenaConverterFactory +import eu.neverblink.jelly.core.RdfEncoder +import eu.neverblink.jelly.core.proto.v1.{PhysicalStreamType, RdfStreamFrame, RdfStreamOptions} +import org.apache.jena.graph.Triple +import org.apache.jena.riot.system.StreamRDF +import org.apache.jena.sparql.core.Quad + +import java.io.OutputStream + +/** Writes a Jelly-RDF 1.2 stream of RDF Messages, ending a message when [[endMessage]] is called. + * + * @param options + * stream options, with the physical type set and the stream type MESSAGES + * @param frameSize + * target number of statements per frame + * @param delimited + * whether the output is delimited. If not, there can be only one frame. + */ +final class JellyMessageWriter( + options: RdfStreamOptions, + frameSize: Int, + delimited: Boolean, + out: OutputStream, +) extends StreamRDF: + private var frameCount = 0 + private val encoder = JenaConverterFactory.getInstance().encoder( + RdfEncoder.Params.of( + options, + if delimited then frameSize else RdfEncoder.MAX_FRAME_SIZE, + writeFrame, + ), + ) + private val triples = options.getPhysicalType == PhysicalStreamType.TRIPLES + + private def writeFrame(frame: RdfStreamFrame): Unit = + frameCount += 1 + if delimited then frame.writeDelimitedTo(out) + else if frameCount > 1 then + throw CriticalException( + "The output does not fit in a single frame, so it cannot be written as non-delimited " + + "output.", + ) + else frame.writeTo(out) + + override def start(): Unit = () + + override def triple(triple: Triple): Unit = + encoder.handleTriple(triple.getSubject, triple.getPredicate, triple.getObject) + + override def quad(quad: Quad): Unit = + if triples then throw CriticalException("Cannot write quads to a Jelly TRIPLES stream.") + encoder.handleQuad(quad.getSubject, quad.getPredicate, quad.getObject, quad.getGraph) + + /** Ends the current message: the statements since the previous one make up one message. */ + def endMessage(): Unit = encoder.handleMessageEnd() + + override def base(base: String): Unit = () + + override def version(version: String): Unit = () + + override def prefix(prefix: String, iri: String): Unit = () + + override def finish(): Unit = + encoder.flush() + out.flush() diff --git a/src/main/scala/eu/neverblink/jelly/cli/util/jena/riot/JellyStreamWriterGraphs.scala b/src/main/scala/eu/neverblink/jelly/cli/util/jena/riot/JellyStreamWriterGraphs.scala index 6db5157..4aab8d6 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/util/jena/riot/JellyStreamWriterGraphs.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/util/jena/riot/JellyStreamWriterGraphs.scala @@ -10,6 +10,7 @@ import org.apache.jena.riot.system.StreamRDF import org.apache.jena.sparql.core.Quad import java.io.OutputStream +import scala.annotation.nowarn /** A stream writer that writes RDF data in Jelly format, using the GRAPHS physical stream type. * @@ -20,6 +21,8 @@ final class JellyStreamWriterGraphs(opt: JellyFormatVariant, out: OutputStream) private val buffer = RowBuffer.newLazyImmutable() // We don't set any options here – it is the responsibility of the caller to set // a valid stream type (GRAPHS). + // The Jelly-RDF 1.1 encoder is deprecated, but still needed for --opt.jelly-version=1.1 + @nowarn("cat=deprecation") private val encoder = JenaConverterFactory.getInstance().encoder( ProtoEncoder.Params.of( opt.getOptions, diff --git a/src/main/scala/eu/neverblink/jelly/cli/util/jena/riot/JellyWriterUtil.scala b/src/main/scala/eu/neverblink/jelly/cli/util/jena/riot/JellyWriterUtil.scala index f025fb2..ac6a80c 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/util/jena/riot/JellyWriterUtil.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/util/jena/riot/JellyWriterUtil.scala @@ -12,13 +12,14 @@ object JellyWriterUtil: /** Creates a StreamRDF that writes Jelly-RDF with the given options. * - * If the physical type is unspecified, the writer picks TRIPLES or QUADS based on the first - * statement it gets. + * The writer writes Jelly-RDF 1.2 by default, unless the protocol version in the options is set + * to 1 or 2. If the physical type is unspecified, the writer picks TRIPLES or QUADS based on the + * first statement it gets. * * @param jellyOpt * stream options of the output * @param frameSize - * target number of rows per frame + * target number of statements (Jelly-RDF 1.2) or rows (Jelly-RDF 1.1) per frame * @param enableNamespaceDeclarations * whether to write namespace declarations * @param delimited @@ -34,6 +35,7 @@ object JellyWriterUtil: out: OutputStream, ): StreamRDF = jellyOpt.getPhysicalType match + // Only in Jelly-RDF 1.1, Jelly-RDF 1.2 has no GRAPHS streams case PhysicalStreamType.GRAPHS => JellyStreamWriterGraphs( JellyFormatVariant diff --git a/src/main/scala/eu/neverblink/jelly/cli/util/jena/riot/RiotParserUtil.scala b/src/main/scala/eu/neverblink/jelly/cli/util/jena/riot/RiotParserUtil.scala index 1c757c1..5c1cec3 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/util/jena/riot/RiotParserUtil.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/util/jena/riot/RiotParserUtil.scala @@ -4,7 +4,9 @@ import eu.neverblink.jelly.cli.command.rdf.util.RdfFormat import org.apache.jena.irix.IRIs import org.apache.jena.riot.lang.LabelToNode import org.apache.jena.riot.{RDFParser, RDFParserRegistry, RIOT} -import org.apache.jena.riot.system.StreamRDF +import org.apache.jena.graph.Triple +import org.apache.jena.riot.system.{StreamRDF, StreamRDFWrapper} +import org.apache.jena.sparql.core.Quad import java.io.InputStream @@ -15,8 +17,9 @@ object RiotParserUtil: resolveIris: Boolean, format: RdfFormat.Jena, source: InputStream, - output: StreamRDF, + sink: StreamRDF, ): Unit = { + val output = SkipFinishOnFailure(sink) // Only really enable IRI resolution if the format supports it if resolveIris && format.supportsBaseIri then // Parser with full IRI resolution. @@ -38,3 +41,21 @@ object RiotParserUtil: .create(format.jenaLang, FastParserProfile()) .read(source, "", format.jenaLang.getContentType, output, RIOT.getContext) } + +/** Jena's parsers call finish() on the output even if the output failed. A Jelly writer that failed + * throws again in finish(), which would hide the first error, so finish() is skipped then. + */ +private final class SkipFinishOnFailure(out: StreamRDF) extends StreamRDFWrapper(out): + private var failed = false + + private def guard(f: => Unit): Unit = + try f + catch + case e: Throwable => + failed = true + throw e + + override def triple(triple: Triple): Unit = guard(super.triple(triple)) + override def quad(quad: Quad): Unit = guard(super.quad(quad)) + override def prefix(prefix: String, iri: String): Unit = guard(super.prefix(prefix, iri)) + override def finish(): Unit = if !failed then super.finish() diff --git a/src/test-serial/scala/eu/neverblink/jelly/cli/command/rdf/TermValidationSpec.scala b/src/test-serial/scala/eu/neverblink/jelly/cli/command/rdf/TermValidationSpec.scala index dfd4a92..1a3b0f3 100644 --- a/src/test-serial/scala/eu/neverblink/jelly/cli/command/rdf/TermValidationSpec.scala +++ b/src/test-serial/scala/eu/neverblink/jelly/cli/command/rdf/TermValidationSpec.scala @@ -127,7 +127,7 @@ class TermValidationSpec extends AnyWordSpec, Matchers, TestFixtureHelper: val newIn = new ByteArrayInputStream(RdfToJelly.getOutBytes) val frame = readJellyFile(newIn) frame.size should be(1) - frame.head.getRows.size() should be > 3 + frame.head.getColumns.getRowCount should be > 0 err shouldBe empty } @@ -141,7 +141,7 @@ class TermValidationSpec extends AnyWordSpec, Matchers, TestFixtureHelper: val newIn = new ByteArrayInputStream(RdfToJelly.getOutBytes) val frame = readJellyFile(newIn) frame.size should be(1) - frame.head.getRows.size() should be > 3 + frame.head.getColumns.getRowCount should be > 0 err shouldBe empty } diff --git a/src/test/scala/eu/neverblink/jelly/cli/command/helpers/DataGenHelper.scala b/src/test/scala/eu/neverblink/jelly/cli/command/helpers/DataGenHelper.scala index b2692d2..bb2d6b3 100644 --- a/src/test/scala/eu/neverblink/jelly/cli/command/helpers/DataGenHelper.scala +++ b/src/test/scala/eu/neverblink/jelly/cli/command/helpers/DataGenHelper.scala @@ -5,7 +5,8 @@ import eu.neverblink.jelly.cli.util.io.ProtoText import eu.neverblink.jelly.core.proto.v1.RdfStreamFrame import org.apache.jena.query.{Dataset, DatasetFactory} import org.apache.jena.rdf.model.{Model, ModelFactory, ResourceFactory} -import org.apache.jena.riot.{Lang, RDFDataMgr, RDFLanguages} +import org.apache.jena.riot.{Lang, RDFDataMgr, RDFLanguages, RDFWriter} +import eu.neverblink.jelly.core.{JellyConstants, JellyOptions} import java.io.{ByteArrayInputStream, ByteArrayOutputStream} @@ -62,10 +63,18 @@ object DataGenHelper: * @return * String */ - def generateJellyBytes(nTriples: Int): Array[Byte] = + def generateJellyBytes(nTriples: Int, rowLayout: Boolean = false): Array[Byte] = val model = generateTripleModel(nTriples) val outputStream = new ByteArrayOutputStream() - RDFDataMgr.write(outputStream, model, JellyLanguage.JELLY) + if rowLayout then + // Jelly-RDF 1.1, with generalized statements and RDF-star allowed + RDFWriter.create().lang(JellyLanguage.JELLY).source(model) + .set( + JellyLanguage.SYMBOL_STREAM_OPTIONS, + JellyOptions.BIG_ALL_FEATURES.clone().setVersion(JellyConstants.PROTO_VERSION_1_1_X), + ) + .output(outputStream) + else RDFDataMgr.write(outputStream, model, JellyLanguage.JELLY) outputStream.toByteArray /** Generate a Jelly frame in the Text Format. @@ -74,8 +83,8 @@ object DataGenHelper: * @return * String */ - def generateJellyText(nTriples: Int): String = - val bytes = generateJellyBytes(nTriples) + def generateJellyText(nTriples: Int, rowLayout: Boolean = false): String = + val bytes = generateJellyBytes(nTriples, rowLayout) val frame = RdfStreamFrame.parseDelimitedFrom(ByteArrayInputStream(bytes)) ProtoText.print(RdfStreamFrame.getDescriptor, frame.toByteArray) diff --git a/src/test/scala/eu/neverblink/jelly/cli/command/helpers/RdfSparqlTestData.scala b/src/test/scala/eu/neverblink/jelly/cli/command/helpers/RdfSparqlTestData.scala index a11aba6..2284913 100644 --- a/src/test/scala/eu/neverblink/jelly/cli/command/helpers/RdfSparqlTestData.scala +++ b/src/test/scala/eu/neverblink/jelly/cli/command/helpers/RdfSparqlTestData.scala @@ -1,18 +1,22 @@ package eu.neverblink.jelly.cli.command.helpers -import eu.neverblink.jelly.cli.command.rdf.util.JellyUtil +import eu.neverblink.jelly.cli.command.rdf.util.{JellyUtil, RdfJellySerializationOptions} import eu.neverblink.jelly.cli.command.sparql.SparqlSerDesSpec +import eu.neverblink.jelly.cli.util.jena.RdfSparqlConverter import eu.neverblink.jelly.cli.util.jena.riot.JellyWriterUtil +import eu.neverblink.jelly.convert.jena.JenaConverterFactory import eu.neverblink.jelly.convert.jena.riot.JellyLanguage +import eu.neverblink.jelly.core.RdfHandler.AnyStatementHandler import eu.neverblink.jelly.convert.jena.sparql.JellySparqlLanguage -import eu.neverblink.jelly.core.JellyOptions -import eu.neverblink.jelly.core.proto.v1.{PhysicalStreamType, RdfStreamOptions} +import eu.neverblink.jelly.core.{JellyConstants, JellyOptions} +import eu.neverblink.jelly.core.proto.v1.{PhysicalStreamType, RdfStreamOptions, RdfVersion} import org.apache.jena.graph.{Node, NodeFactory, Triple} import org.apache.jena.query.ResultSetFactory import org.apache.jena.riot.RDFParser import org.apache.jena.sparql.core.{DatasetGraphFactory, Quad} import java.io.{ByteArrayInputStream, ByteArrayOutputStream} +import scala.collection.mutable.ArrayBuffer import scala.jdk.CollectionConverters.* /** Test data and helpers for converting between Jelly-RDF and Jelly-SPARQL. */ @@ -36,17 +40,32 @@ object RdfSparqlTestData: /** The triples as quads in the default graph. */ val triplesAsQuads: Set[Quad] = triples.map(Quad.create(Quad.defaultGraphIRI, _)).toSet - /** Writes the statements as Jelly-RDF with the given physical type. */ + /** Writes the statements as Jelly-RDF with the given physical type. + * + * @param rowLayout + * whether to write Jelly-RDF 1.1, rather than 1.2. GRAPHS streams are always Jelly-RDF 1.1. + * @param rdfStar + * whether triple terms are allowed, in Jelly-RDF 1.1 + * @param rdfVersion + * the RDF version, in Jelly-RDF 1.2 + */ def writeRdf( physicalType: PhysicalStreamType, triples: Seq[Triple] = Nil, quads: Seq[Quad] = Nil, rdfStar: Boolean = true, + rdfVersion: RdfVersion = RdfVersion.RDF_VERSION_UNSPECIFIED, rowsPerFrame: Int = 256, + rowLayout: Boolean = false, ): Array[Byte] = val out = ByteArrayOutputStream() + val options = + if rowLayout || physicalType == PhysicalStreamType.GRAPHS then + JellyOptions.SMALL_ALL_FEATURES.clone().setRdfStar(rdfStar) + .setVersion(JellyConstants.PROTO_VERSION_1_1_X) + else JellyOptions.SMALL_STRICT.clone().setRdfVersion(rdfVersion) val writer = JellyWriterUtil.createWriter( - JellyOptions.SMALL_ALL_FEATURES.clone().setPhysicalType(physicalType).setRdfStar(rdfStar), + options.setPhysicalType(physicalType), rowsPerFrame, false, true, @@ -68,8 +87,38 @@ object RdfSparqlTestData: /** The stream options of a Jelly-RDF stream. */ def rdfStreamOptions(jelly: Array[Byte]): RdfStreamOptions = - JellyUtil.iterateRdfStream(ByteArrayInputStream(jelly)).next().getRows.iterator().next() - .getOptions + JellyUtil.requireFirstOptions(JellyUtil.iterateRdfStream(ByteArrayInputStream(jelly)).buffered) + + /** Converts Jelly-SPARQL to Jelly-RDF, just like rdf to-jelly does with no options. */ + def sparqlToRdfMessages(sparql: Array[Byte]): Array[Byte] = + val out = ByteArrayOutputStream() + RdfSparqlConverter.sparqlToRdf( + ByteArrayInputStream(sparql), + out, + RdfJellySerializationOptions.defaultOptions, + streamTypeSet = false, + 256, + delimited = true, + ) + out.toByteArray + + /** Reads a Jelly-RDF stream of RDF Messages into one set of quads per message. */ + def readRdfMessages(jelly: Array[Byte]): Seq[Set[Quad]] = + val messages = ArrayBuffer[Set[Quad]]() + var current = Set[Quad]() + val handler = new AnyStatementHandler[Node]: + override def handleTriple(s: Node, p: Node, o: Node): Unit = + current += Quad.create(Quad.defaultGraphIRI, s, p, o) + override def handleQuad(s: Node, p: Node, o: Node, g: Node): Unit = + current += Quad.create(if Quad.isDefaultGraph(g) then Quad.defaultGraphIRI else g, s, p, o) + override def handleMessageEnd(): Unit = + messages += current + current = Set() + val decoder = JenaConverterFactory.getInstance() + .anyStatementDecoder(handler, JellyOptions.DEFAULT_SUPPORTED_OPTIONS) + JellyUtil.iterateRdfStream(ByteArrayInputStream(jelly)).foreach(decoder.ingestFrame) + // The end of the stream ends the last message + (messages :+ current).toSeq /** Reads Jelly-SPARQL into the variables and rows, with each row as a map of the bound values. */ def readSparql(jelly: Array[Byte]): (Seq[String], Seq[Map[String, Node]]) = diff --git a/src/test/scala/eu/neverblink/jelly/cli/command/helpers/TestFixtureHelper.scala b/src/test/scala/eu/neverblink/jelly/cli/command/helpers/TestFixtureHelper.scala index 0abb120..5703232 100644 --- a/src/test/scala/eu/neverblink/jelly/cli/command/helpers/TestFixtureHelper.scala +++ b/src/test/scala/eu/neverblink/jelly/cli/command/helpers/TestFixtureHelper.scala @@ -3,7 +3,7 @@ package eu.neverblink.jelly.cli.command.helpers import eu.neverblink.jelly.cli.util.jena.JenaSystemOptions import eu.neverblink.jelly.cli.util.jena.riot.CliRiot import eu.neverblink.jelly.convert.jena.riot.{JellyFormatVariant, JellyLanguage} -import eu.neverblink.jelly.core.JellyOptions +import eu.neverblink.jelly.core.{JellyConstants, JellyOptions} import org.apache.jena.graph.Triple import org.apache.jena.riot.{Lang, RDFDataMgr, RDFFormat, RDFLanguages, RDFWriter, RIOT} import org.apache.jena.shared.impl.JenaParameters @@ -62,7 +62,7 @@ trait TestFixtureHelper extends BeforeAndAfterAll: val tempFile = Files.createTempFile(tmpDir, randomUUID.toString, f".${extension}") val graph = GraphFactory.createGraphMem() content.foreach(graph.add) - Using(new FileOutputStream(tempFile.toFile)) { fileOutputStream => + Using.resource(new FileOutputStream(tempFile.toFile)) { fileOutputStream => RDFDataMgr.write(fileOutputStream, graph, jenaLang) } try { @@ -75,9 +75,9 @@ trait TestFixtureHelper extends BeforeAndAfterAll: testCode(tempFile.toString) } finally { tempFile.toFile.delete() } - def withFullJellyTextFile(testCode: (String) => Any): Unit = + def withFullJellyTextFile(testCode: (String) => Any, rowLayout: Boolean = false): Unit = val tempFile = Files.createTempFile(tmpDir, randomUUID.toString, ".jelly.txt") - val text = DataGenHelper.generateJellyText(testCardinality) + val text = DataGenHelper.generateJellyText(testCardinality, rowLayout) Files.write(tempFile, text.getBytes) try { testCode(tempFile.toString) @@ -99,7 +99,14 @@ trait TestFixtureHelper extends BeforeAndAfterAll: else testCode(filePath.toString) } - def withFullJellyFile(testCode: (String) => Any, frameSize: Int = 256): Unit = + /** @param rowLayout + * whether to write Jelly-RDF 1.1, rather than 1.2 + */ + def withFullJellyFile( + testCode: (String) => Any, + frameSize: Int = 256, + rowLayout: Boolean = false, + ): Unit = val extension = getFileExtension(JellyLanguage.JELLY) val tempFile = Files.createTempFile(tmpDir, randomUUID.toString, f".${extension}") val customFormat = new RDFFormat( @@ -109,7 +116,8 @@ trait TestFixtureHelper extends BeforeAndAfterAll: val writerContext = RIOT.getContext.copy.set( JellyLanguage.SYMBOL_STREAM_OPTIONS, - JellyOptions.SMALL_ALL_FEATURES.clone().setStreamName("Stream"), + JellyOptions.SMALL_ALL_FEATURES.clone().setStreamName("Stream") + .setVersion(if rowLayout then JellyConstants.PROTO_VERSION_1_1_X else 0), ) val model = DataGenHelper.generateTripleModel(testCardinality) diff --git a/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfFromJellySpec.scala b/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfFromJellySpec.scala index a70ea26..2cb2b44 100644 --- a/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfFromJellySpec.scala +++ b/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfFromJellySpec.scala @@ -5,8 +5,9 @@ import eu.neverblink.jelly.cli.* import eu.neverblink.jelly.cli.command.helpers.* import eu.neverblink.jelly.cli.command.rdf.util.{RdfFormat, RdfJellySerializationOptions} import eu.neverblink.jelly.convert.jena.riot.{JellyFormat, JellyLanguage} -import eu.neverblink.jelly.core.proto.v1.{PhysicalStreamType, RdfStreamFrame} -import eu.neverblink.jelly.core.{JellyOptions, JellyTranscoderFactory} +import eu.neverblink.jelly.cli.command.sparql.SparqlSerDesSpec +import eu.neverblink.jelly.core.proto.v1.{PhysicalStreamType, RdfStreamFrame, RdfStreamType} +import eu.neverblink.jelly.core.{JellyConstants, JellyOptions, JellyTranscoderFactory} import org.apache.jena.query.DatasetFactory import org.apache.jena.rdf.model.ModelFactory import org.apache.jena.riot.system.StreamRDFLib @@ -45,6 +46,19 @@ class RdfFromJellySpec extends AnyWordSpec with Matchers with TestFixtureHelper: os.toByteArray } + // A stream of 3 RDF Messages, the second one empty + private lazy val messageStream: Array[Byte] = { + import RdfSparqlTestData.* + val sparql = SparqlSerDesSpec.writePunctuated( + Seq( + sparqlJson(Seq("s", "p", "o"), Seq(statementRow("a", "p", "1"))), + sparqlJson(Seq("s", "p", "o"), Nil), + sparqlJson(Seq("s", "p", "o"), Seq(statementRow("b", "p", "2"))), + ), + ) + sparqlToRdfMessages(sparql) + } + "rdf from-jelly command" should { "re-encode Jelly with the default options of rdf to-jelly" when { "the format is given explicitly" in withFullJellyFile( @@ -53,12 +67,12 @@ class RdfFromJellySpec extends AnyWordSpec with Matchers with TestFixtureHelper: val bytes = RdfFromJelly.getOutBytes val frames = RdfToJellySpec.readJellyFile(ByteArrayInputStream(bytes)) val options = frames.head.getRows.asScala.head.getOptions - val expected = RdfJellySerializationOptions.defaultFor(RdfFormat.JellyBinary) + val expected = RdfJellySerializationOptions.defaultOptions options.getMaxNameTableSize should be(expected.getMaxNameTableSize) - options.getGeneralizedStatements should be(true) - options.getRdfStar should be(true) + options.getVersion should be(JellyConstants.PROTO_VERSION_1_2_X) + options.getStreamType should be(RdfStreamType.FLAT) options.getPhysicalType should be(PhysicalStreamType.TRIPLES) - // The input has frames of 10 rows, which are kept as they are + // The input has frames of 10 statements, which are kept as they are frames.size should be > 1 RdfToJellySpec.translateJellyBack(ByteArrayInputStream(bytes)) .isIsomorphicWith(DataGenHelper.generateTripleModel(testCardinality)) should be(true) @@ -66,6 +80,29 @@ class RdfFromJellySpec extends AnyWordSpec with Matchers with TestFixtureHelper: frameSize = 10, ) + "the input is Jelly-RDF 1.1" in withFullJellyFile( + j => { + RdfFromJelly.runTestCommand(List("rdf", "from-jelly", j, "--out-format", "jelly")) + val bytes = RdfFromJelly.getOutBytes + val frames = RdfToJellySpec.readJellyFile(ByteArrayInputStream(bytes)) + frames.head.getRows.asScala.head.getOptions.getVersion should be( + JellyConstants.PROTO_VERSION_1_2_X, + ) + RdfToJellySpec.translateJellyBack(ByteArrayInputStream(bytes)) + .isIsomorphicWith(DataGenHelper.generateTripleModel(testCardinality)) should be(true) + }, + rowLayout = true, + ) + + "the input is a stream of RDF Messages, which stays one" in { + val messages = RdfSparqlTestData.readRdfMessages(messageStream) + RdfFromJelly.setStdIn(ByteArrayInputStream(messageStream)) + RdfFromJelly.runTestCommand(List("rdf", "from-jelly", "--out-format", "jelly")) + val bytes = RdfFromJelly.getOutBytes + RdfSparqlTestData.rdfStreamOptions(bytes).getStreamType should be(RdfStreamType.MESSAGES) + RdfSparqlTestData.readRdfMessages(bytes) should be(messages) + } + "the format is inferred from the output file name" in withFullJellyFile { j => withEmptyJellyFile { q => RdfFromJelly.runTestCommand(List("rdf", "from-jelly", j, "--to", q)) @@ -201,17 +238,16 @@ class RdfFromJellySpec extends AnyWordSpec with Matchers with TestFixtureHelper: | options { | stream_name: "Stream" | physical_type: PHYSICAL_STREAM_TYPE_TRIPLES - | generalized_statements: true - | rdf_star: true | max_name_table_size: 128 | max_prefix_table_size: 16 | max_datatype_table_size: 16 - | logical_type: LOGICAL_STREAM_TYPE_FLAT_TRIPLES - | version: 1 + | version: 3 | } - |}""".stripMargin + |} + |columns { + | row_count: 33""".stripMargin out should include(outString) - "rows".r.findAllIn(out).length should be(70) + "rows".r.findAllIn(out).length should be(1) "http://example.org/predicate/".r.findAllIn(out).length should be(1) } @@ -236,17 +272,16 @@ class RdfFromJellySpec extends AnyWordSpec with Matchers with TestFixtureHelper: | options { | stream_name: "Stream" | physical_type: PHYSICAL_STREAM_TYPE_TRIPLES - | generalized_statements: true - | rdf_star: true | max_name_table_size: 128 | max_prefix_table_size: 16 | max_datatype_table_size: 16 - | logical_type: LOGICAL_STREAM_TYPE_FLAT_TRIPLES - | version: 1 + | version: 3 | } - |}""".stripMargin + |} + |columns { + | row_count: 33""".stripMargin inTxt should include(outString) - "rows".r.findAllIn(inTxt).length should be(70) + "rows".r.findAllIn(inTxt).length should be(1) "http://example.org/predicate/".r.findAllIn(inTxt).length should be(1) } } @@ -266,7 +301,7 @@ class RdfFromJellySpec extends AnyWordSpec with Matchers with TestFixtureHelper: out should include("# Frame 3") out should include("# Frame 4") out should include("# Frame 5") - "rows".r.findAllIn(out).length should be(3 * testCardinality) + s"row_count: $testCardinality".r.findAllIn(out).length should be(3) } } @@ -443,7 +478,7 @@ class RdfFromJellySpec extends AnyWordSpec with Matchers with TestFixtureHelper: withEmptyJellyFile { j => RDFWriter .source(testDs) - .format(JellyFormat.JELLY_BIG_ALL_FEATURES) + .format(JellyFormat.JELLY_BIG_STRICT) .build() .output(FileOutputStream(File(j))) @@ -476,7 +511,7 @@ class RdfFromJellySpec extends AnyWordSpec with Matchers with TestFixtureHelper: withEmptyJellyFile { j => RDFWriter .source(testDs) - .format(JellyFormat.JELLY_BIG_ALL_FEATURES) + .format(JellyFormat.JELLY_BIG_STRICT) .build() .output(FileOutputStream(File(j))) @@ -524,8 +559,9 @@ class RdfFromJellySpec extends AnyWordSpec with Matchers with TestFixtureHelper: } } - "use the default options, with the RDF version inferred from the input" in { - val rdf = writeRdf(PhysicalStreamType.TRIPLES, triples, rdfStar = false) + "use the default options, with the RDF version of the input" in { + val rdf = + writeRdf(PhysicalStreamType.TRIPLES, triples, rdfVersion = RdfVersion.RDF_VERSION_1_1) RdfFromJelly.setStdIn(ByteArrayInputStream(rdf)) RdfFromJelly.runTestCommand(List("rdf", "from-jelly", "--out-format", "jelly-sparql")) val options = readFrames(RdfFromJelly.getOutBytes).head.getOptions diff --git a/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfInspectSpec.scala b/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfInspectSpec.scala index 0782fd0..bafa1e6 100644 --- a/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfInspectSpec.scala +++ b/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfInspectSpec.scala @@ -2,16 +2,19 @@ package eu.neverblink.jelly.cli.command.rdf import com.google.protobuf.ByteString import eu.neverblink.jelly.cli.{ExitException, InvalidJellyFile} -import eu.neverblink.jelly.cli.command.helpers.TestFixtureHelper -import eu.neverblink.jelly.core.JellyOptions -import eu.neverblink.jelly.core.proto.v1.{RdfStreamFrame, RdfStreamRow} +import eu.neverblink.jelly.cli.command.helpers.{RdfSparqlTestData, TestFixtureHelper} +import eu.neverblink.jelly.cli.command.sparql.SparqlSerDesSpec +import eu.neverblink.jelly.core.{JellyConstants, JellyOptions} +import eu.neverblink.jelly.cli.util.jena.riot.JellyWriterUtil +import eu.neverblink.jelly.core.proto.v1.{PhysicalStreamType, RdfStreamFrame, RdfStreamRow} +import org.apache.jena.graph.{NodeFactory, Triple} import scala.jdk.CollectionConverters.* import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec import org.yaml.snakeyaml.Yaml -import java.io.ByteArrayInputStream +import java.io.{ByteArrayInputStream, ByteArrayOutputStream} import java.util class RdfInspectSpec extends AnyWordSpec with Matchers with TestFixtureHelper: @@ -19,8 +22,158 @@ class RdfInspectSpec extends AnyWordSpec with Matchers with TestFixtureHelper: protected val tripleTerms: Seq[String] = Seq("subject", "predicate", "object") protected val tripleNodes: Seq[String] = Seq("iri", "bnode", "literal", "triple") + /** Most tests here are about Jelly-RDF 1.1 streams, see "for Jelly-RDF 1.2 streams" below. */ + private def withRowLayoutJellyFile(testCode: String => Any, frameSize: Int = 256): Unit = + withFullJellyFile(testCode, frameSize, rowLayout = true) + + private def inspect(args: String*): util.Map[String, Any] = + val (out, _) = RdfInspect.runTestCommand(List("rdf", "inspect") ++ args) + new Yaml().load(out).asInstanceOf[util.Map[String, Any]] + + private def aggregate(parsed: util.Map[String, Any]): util.Map[String, Any] = + parsed.get("frames").asInstanceOf[util.Map[String, Any]] + + private def perFrame(parsed: util.Map[String, Any]): Seq[util.Map[String, Any]] = + parsed.get("frames").asInstanceOf[util.List[util.Map[String, Any]]].asScala.toSeq + + private def map(m: util.Map[String, Any], key: String): util.Map[String, Any] = + m.get(key).asInstanceOf[util.Map[String, Any]] + + "rdf inspect command, for Jelly-RDF 1.2 streams" should { + "print the stream options of Jelly-RDF 1.2" in withFullJellyFile { j => + val options = map(inspect(j), "stream_options") + options.get("max_name_table_size") should be(128) + options.get("physical_type") should be("TRIPLES (1)") + options.get("stream_type") should be("FLAT (0)") + options.get("rdf_version") should be("RDF_VERSION_UNSPECIFIED (0)") + options.get("version") should be(JellyConstants.PROTO_VERSION_1_2_X) + options.containsKey("logical_type") should be(false) + options.containsKey("generalized_statements") should be(false) + } + + "count statements, lookup entries, and column values" in withFullJellyFile( + j => { + val frames = aggregate(inspect(j)) + frames.get("frame_count") should be(3) + frames.get("statement_count") should be(testCardinality) + frames.get("option_count") should be(1) + frames.get("name_count").asInstanceOf[Int] should be > 0 + // Subject, predicate, and object are all IRIs + frames.get("iri_value_count") should be(3 * testCardinality) + frames.get("literal_value_count") should be(0) + frames.containsKey("triple_count") should be(false) + frames.containsKey("message_end_count") should be(false) + }, + frameSize = 15, + ) + + "count statements per frame" in withFullJellyFile( + j => { + val frames = perFrame(inspect("--per-frame", j)) + frames.map(_.get("statement_count")) should be(Seq(15, 15, 3)) + frames.map(_.get("frame_index")) should be(Seq(0, 1, 2)) + }, + frameSize = 15, + ) + + "print the values of each column with --detail all" in withFullJellyFile( + j => { + val frames = aggregate(inspect("--detail", "all", j)) + for column <- tripleTerms do + map(frames, column).get("iri_value_count") should be(testCardinality) + map(frames, column).containsKey("bnode_value_count") should be(false) + frames.containsKey("graph") should be(false) + val first = perFrame(inspect("--per-frame", "--detail", "all", j)).head + map(first, "subject").get("iri_value_count") should be(15) + }, + frameSize = 15, + ) + + "print the values by kind with --detail node" in withFullJellyFile { j => + val details = map(aggregate(inspect("--detail", "node", j)), "node_details") + details.get("iri_value_count") should be(3 * testCardinality) + } + + "print the values by column with --detail term" in withFullJellyFile { j => + val details = map(aggregate(inspect("--detail", "term", j)), "term_details") + details.get("subject_count") should be(testCardinality) + details.get("graph_count") should be(0) + } + + "print size statistics" in withFullJellyFile { j => + val frames = aggregate(inspect("--size", "--detail", "all", j)) + // The number of statements is not a size + frames.get("statement_count") should be(testCardinality) + frames.get("frame_size").asInstanceOf[Int] should be > testCardinality + frames.get("iri_value_size").asInstanceOf[Int] should be > 0 + map(frames, "subject").get("iri_value_size").asInstanceOf[Int] should be > 0 + } + + "print the values of the graph column of a QUADS stream" in { + import RdfSparqlTestData.* + RdfInspect.setStdIn(ByteArrayInputStream(writeRdf(PhysicalStreamType.QUADS, quads = quads))) + val parsed = inspect("--detail", "all") + map(parsed, "stream_options").get("physical_type") should be("QUADS (2)") + val frames = aggregate(parsed) + frames.get("statement_count") should be(quads.size) + // Two named graphs, and one quad in the default graph, which has no value + map(frames, "graph").get("iri_value_count") should be(2) + RdfInspect.setStdIn(ByteArrayInputStream(writeRdf(PhysicalStreamType.QUADS, quads = quads))) + map(aggregate(inspect("--detail", "term")), "term_details").get("graph_count") should be(2) + } + + "measure the size of triple terms" in { + import RdfSparqlTestData.* + val withTripleTerm = Triple.create( + iri("a"), + iri("says"), + NodeFactory.createTripleTerm(triples.head), + ) + val rdf = writeRdf(PhysicalStreamType.TRIPLES, Seq(withTripleTerm)) + RdfInspect.setStdIn(ByteArrayInputStream(rdf)) + aggregate(inspect()).get("triple_term_value_count") should be(1) + RdfInspect.setStdIn(ByteArrayInputStream(rdf)) + val frames = aggregate(inspect("--size", "--detail", "all")) + frames.get("triple_term_value_size").asInstanceOf[Int] should be > 0 + map(frames, "object").get("triple_term_value_size") should be( + frames.get("triple_term_value_size"), + ) + } + + "count namespace declarations" in { + val out = ByteArrayOutputStream() + val writer = JellyWriterUtil.createWriter( + JellyOptions.SMALL_STRICT.clone().setPhysicalType(PhysicalStreamType.TRIPLES), + 256, + enableNamespaceDeclarations = true, + delimited = true, + out, + ) + writer.prefix("ex", "http://example.org/") + writer.prefix("foaf", "http://xmlns.com/foaf/0.1/") + writer.triple(RdfSparqlTestData.triples.head) + writer.finish() + RdfInspect.setStdIn(ByteArrayInputStream(out.toByteArray)) + aggregate(inspect()).get("namespace_count") should be(2) + RdfInspect.setStdIn(ByteArrayInputStream(out.toByteArray)) + aggregate(inspect("--size")).get("namespace_size").asInstanceOf[Int] should be > 0 + } + + "count the message ends of a stream of RDF Messages" in { + import RdfSparqlTestData.* + val result = sparqlJson(Seq("s", "p", "o"), Seq(statementRow("a", "p", "1"))) + val sparql = SparqlSerDesSpec.writePunctuated(Seq(result, result, result)) + RdfInspect.setStdIn(ByteArrayInputStream(sparqlToRdfMessages(sparql))) + val parsed = inspect() + map(parsed, "stream_options").get("stream_type") should be("MESSAGES (1)") + // The end of the stream ends the last message + aggregate(parsed).get("message_end_count") should be(2) + aggregate(parsed).get("statement_count") should be(3) + } + } + "rdf inspect command" should { - "be able to return aggregate of all frames as a valid Yaml" in withFullJellyFile { j => + "be able to return aggregate of all frames as a valid Yaml" in withRowLayoutJellyFile { j => val (out, err) = RdfInspect.runTestCommand(List("rdf", "inspect", j)) val yaml = new Yaml() val parsed = yaml.load(out).asInstanceOf[java.util.Map[String, Any]] @@ -32,7 +185,7 @@ class RdfInspectSpec extends AnyWordSpec with Matchers with TestFixtureHelper: frames.get("triple_count") should be(testCardinality) } - "be able to return all frames separately as a valid Yaml" in withFullJellyFile( + "be able to return all frames separately as a valid Yaml" in withRowLayoutJellyFile( testCode = { j => val (out, err) = RdfInspect.runTestCommand(List("rdf", "inspect", "--per-frame", j)) val yaml = new Yaml() @@ -47,7 +200,7 @@ class RdfInspectSpec extends AnyWordSpec with Matchers with TestFixtureHelper: frameSize = 15, ) - "handle properly separate frame metrics for a singular frame" in withFullJellyFile { j => + "handle properly separate frame metrics for a singular frame" in withRowLayoutJellyFile { j => val (out, err) = RdfInspect.runTestCommand(List("rdf", "inspect", "--per-frame", j)) val yaml = new Yaml() val parsed = yaml.load(out).asInstanceOf[java.util.Map[String, Any]] @@ -59,7 +212,7 @@ class RdfInspectSpec extends AnyWordSpec with Matchers with TestFixtureHelper: frames.map(_.get("triple_count")).sum should be(testCardinality) } - "handle properly frame count when aggregating multiple frames" in withFullJellyFile( + "handle properly frame count when aggregating multiple frames" in withRowLayoutJellyFile( testCode = { j => val (out, err) = RdfInspect.runTestCommand(List("rdf", "inspect", j)) val yaml = new Yaml() @@ -108,7 +261,7 @@ class RdfInspectSpec extends AnyWordSpec with Matchers with TestFixtureHelper: } "print detailed metrics" when { - "aggregating and flat" in withFullJellyFile( + "aggregating and flat" in withRowLayoutJellyFile( testCode = { j => val (out, err) = RdfInspect.runTestCommand(List("rdf", "inspect", "--detail", "all", j)) val yaml = new Yaml() @@ -123,7 +276,7 @@ class RdfInspectSpec extends AnyWordSpec with Matchers with TestFixtureHelper: frameSize = 15, ) - "per frame and flat" in withFullJellyFile( + "per frame and flat" in withRowLayoutJellyFile( testCode = { j => val (out, err) = RdfInspect.runTestCommand(List("rdf", "inspect", "--per-frame", "--detail", "all", j)) @@ -140,7 +293,7 @@ class RdfInspectSpec extends AnyWordSpec with Matchers with TestFixtureHelper: frameSize = 15, ) - "aggregating and grouping by node" in withFullJellyFile( + "aggregating and grouping by node" in withRowLayoutJellyFile( testCode = { j => val (out, err) = RdfInspect.runTestCommand(List("rdf", "inspect", "--detail", "node", j)) val yaml = new Yaml() @@ -155,7 +308,7 @@ class RdfInspectSpec extends AnyWordSpec with Matchers with TestFixtureHelper: frameSize = 15, ) - "per frame and grouping by node" in withFullJellyFile( + "per frame and grouping by node" in withRowLayoutJellyFile( testCode = { j => val (out, err) = RdfInspect.runTestCommand(List("rdf", "inspect", "--per-frame", "--detail", "node", j)) @@ -172,7 +325,7 @@ class RdfInspectSpec extends AnyWordSpec with Matchers with TestFixtureHelper: frameSize = 15, ) - "aggregating and grouping by term" in withFullJellyFile( + "aggregating and grouping by term" in withRowLayoutJellyFile( testCode = { j => val (out, err) = RdfInspect.runTestCommand(List("rdf", "inspect", "--detail", "term", j)) val yaml = new Yaml() @@ -186,7 +339,7 @@ class RdfInspectSpec extends AnyWordSpec with Matchers with TestFixtureHelper: frameSize = 15, ) - "per frame and grouping by term" in withFullJellyFile( + "per frame and grouping by term" in withRowLayoutJellyFile( testCode = { j => val (out, err) = RdfInspect.runTestCommand(List("rdf", "inspect", "--per-frame", "--detail", "term", j)) @@ -255,7 +408,7 @@ class RdfInspectSpec extends AnyWordSpec with Matchers with TestFixtureHelper: } "print size statistics" when { - "aggregating multiple frames" in withFullJellyFile( + "aggregating multiple frames" in withRowLayoutJellyFile( testCode = { j => val (out, err) = RdfInspect.runTestCommand(List("rdf", "inspect", "--size", j)) val yaml = new Yaml() @@ -271,7 +424,7 @@ class RdfInspectSpec extends AnyWordSpec with Matchers with TestFixtureHelper: frameSize = 15, ) - "per frame" in withFullJellyFile( + "per frame" in withRowLayoutJellyFile( testCode = { j => val (out, err) = RdfInspect.runTestCommand(List("rdf", "inspect", "--per-frame", "--size", j)) @@ -290,7 +443,7 @@ class RdfInspectSpec extends AnyWordSpec with Matchers with TestFixtureHelper: } "print detailed size statistics" when { - "aggregating multiple frames" in withFullJellyFile( + "aggregating multiple frames" in withRowLayoutJellyFile( testCode = { j => val (out, err) = RdfInspect.runTestCommand(List("rdf", "inspect", "--size", "--detail", "all", j)) @@ -306,7 +459,7 @@ class RdfInspectSpec extends AnyWordSpec with Matchers with TestFixtureHelper: frameSize = 15, ) - "per frame" in withFullJellyFile( + "per frame" in withRowLayoutJellyFile( testCode = { j => val (out, err) = RdfInspect.runTestCommand( diff --git a/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJellySpec.scala b/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJellySpec.scala index 8de6219..6643515 100644 --- a/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJellySpec.scala +++ b/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJellySpec.scala @@ -1,11 +1,12 @@ package eu.neverblink.jelly.cli.command.rdf -import eu.neverblink.jelly.cli.command.helpers.{DataGenHelper, TestFixtureHelper} -import eu.neverblink.jelly.cli.command.rdf.util.RdfFormat +import eu.neverblink.jelly.cli.command.helpers.{DataGenHelper, RdfSparqlTestData, TestFixtureHelper} +import eu.neverblink.jelly.cli.command.rdf.util.{JellyUtil, RdfFormat} +import eu.neverblink.jelly.cli.command.sparql.SparqlSerDesSpec import eu.neverblink.jelly.cli.* import eu.neverblink.jelly.convert.jena.riot.JellyLanguage -import eu.neverblink.jelly.core.proto.v1.{LogicalStreamType, PhysicalStreamType, RdfStreamFrame} -import eu.neverblink.jelly.core.JellyOptions +import eu.neverblink.jelly.core.proto.v1.* +import eu.neverblink.jelly.core.{JellyConstants, JellyOptions} import eu.neverblink.jelly.cli.util.io.ProtoText import eu.neverblink.jelly.core.utils.IoUtils import org.apache.jena.irix.IRIs @@ -16,6 +17,7 @@ import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec import java.io.{ByteArrayInputStream, FileInputStream, InputStream} +import java.nio.file.Files import scala.jdk.CollectionConverters.* import scala.util.Using @@ -45,6 +47,12 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers: protected val testCardinality: Int = 33 + private def withBytesFile[T](content: Array[Byte])(testCode: String => T): T = + val file = Files.createTempFile("jelly-cli-test", ".jelly") + Files.write(file, content) + try testCode(file.toString) + finally Files.deleteIfExists(file) + "rdf to-jelly command" should { "re-encode Jelly with the given options" when { "the format is given explicitly" in withFullJellyFile { j => @@ -55,7 +63,7 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers: j, "--in-format", "jelly", - "--opt.max-name-table-size=16", + "--opt.max-name-table-size=200", "--opt.physical-type=QUADS", "--rows-per-frame=5", ), @@ -63,20 +71,31 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers: val bytes = RdfToJelly.getOutBytes val frames = readJellyFile(ByteArrayInputStream(bytes)) val options = frames.head.getRows.asScala.head.getOptions - options.getMaxNameTableSize should be(16) + options.getMaxNameTableSize should be(200) options.getPhysicalType should be(PhysicalStreamType.QUADS) - // Inferred from the input format - options.getGeneralizedStatements should be(true) + options.getVersion should be(JellyConstants.PROTO_VERSION_1_2_X) frames.size should be > 1 translateJellyBack(ByteArrayInputStream(bytes)) .isIsomorphicWith(DataGenHelper.generateTripleModel(testCardinality)) should be(true) } + "the input is Jelly-RDF 1.1, which becomes 1.2" in withFullJellyFile( + j => { + RdfToJelly.runTestCommand(List("rdf", "to-jelly", j)) + val bytes = RdfToJelly.getOutBytes + readJellyFile(ByteArrayInputStream(bytes)).head.getRows.asScala.head.getOptions + .getVersion should be(JellyConstants.PROTO_VERSION_1_2_X) + translateJellyBack(ByteArrayInputStream(bytes)) + .isIsomorphicWith(DataGenHelper.generateTripleModel(testCardinality)) should be(true) + }, + rowLayout = true, + ) + "the format is inferred from the input file name" in withFullJellyFile { j => - RdfToJelly.runTestCommand(List("rdf", "to-jelly", j, "--opt.max-name-table-size=16")) + RdfToJelly.runTestCommand(List("rdf", "to-jelly", j, "--opt.max-name-table-size=200")) val bytes = RdfToJelly.getOutBytes readJellyFile(ByteArrayInputStream(bytes)).head.getRows.asScala.head.getOptions - .getMaxNameTableSize should be(16) + .getMaxNameTableSize should be(200) translateJellyBack(ByteArrayInputStream(bytes)) .isIsomorphicWith(DataGenHelper.generateTripleModel(testCardinality)) should be(true) } @@ -148,11 +167,21 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers: .toSet should be(Set("b1", "b2", "b3")) } + "input stream to output stream, generalized RDF in Jelly-RDF 1.2 (error)" in { + RdfToJelly.setStdIn(getClass.getResourceAsStream("/generalized.nt")) + val e = intercept[ExitException] { + RdfToJelly.runTestCommand(List("rdf", "to-jelly", "--in-format=nt")) + } + e.getCause shouldBe a[JellySerializationError] + e.getCause.getMessage should include("cannot be") + e.getCause.getMessage should include("--opt.jelly-version=1.1") + } + "input stream to output stream, generalized RDF (N-Triples)" in { val inputStream = getClass.getResourceAsStream("/generalized.nt") RdfToJelly.setStdIn(inputStream) val (out, err) = RdfToJelly.runTestCommand( - List("rdf", "to-jelly", "--in-format=nt"), + List("rdf", "to-jelly", "--in-format=nt", "--opt.jelly-version=1.1"), ) val bytes = RdfToJelly.getOutBytes val content = translateJellyBack(new ByteArrayInputStream(bytes)) @@ -166,7 +195,7 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers: val inputStream = getClass.getResourceAsStream("/generalized.nq") RdfToJelly.setStdIn(inputStream) val (out, err) = RdfToJelly.runTestCommand( - List("rdf", "to-jelly", "--in-format=nq"), + List("rdf", "to-jelly", "--in-format=nq", "--opt.jelly-version=1.1"), ) val bytes = RdfToJelly.getOutBytes val ds = DatasetGraphFactory.create() @@ -186,7 +215,13 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers: ) RdfToJelly.setStdIn(inputStream) val (out, err) = RdfToJelly.runTestCommand( - List("rdf", "to-jelly", "--in-format=nq", "--opt.physical-type=GRAPHS"), + List( + "rdf", + "to-jelly", + "--in-format=nq", + "--opt.physical-type=GRAPHS", + "--opt.jelly-version=1.1", + ), ) val ds = DatasetGraphFactory.create() val bytes = RdfToJelly.getOutBytes @@ -219,6 +254,7 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers: "--in-format=nq", "--opt.physical-type=GRAPHS", "--opt.logical-type=DATASETS", + "--opt.jelly-version=1.1", ), ) val ds = DatasetGraphFactory.create() @@ -258,6 +294,7 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers: "--opt.max-prefix-table-size=100", "--opt.max-datatype-table-size=100", "--opt.logical-type=FLAT_QUADS", + "--opt.jelly-version=1.1", "--to", j, ), @@ -288,6 +325,7 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers: "to-jelly", f, "--opt.logical-type=https://w3id.org/stax/ontology#flatQuadStream", + "--opt.jelly-version=1.1", "--to", j, ), @@ -320,6 +358,7 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers: f, "--opt.physical-type=QUADS", "--opt.logical-type=DATASETS", + "--opt.jelly-version=1.1", "--to", j, ), @@ -390,7 +429,7 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers: // TODO: test if the namespace declarations are preserved with Turtle or RDF/XML input. val frames = readJellyFile(new FileInputStream(j)) val opts = frames.head.getRows.asScala.head.getOptions - opts.getVersion should be(2) + opts.getVersion should be(JellyConstants.PROTO_VERSION_1_2_X) } } @@ -441,6 +480,7 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers: f, "--opt.physical-type=TRIPLES", "--opt.logical-type=GRAPHS", + "--opt.jelly-version=1.1", "--to", j, ), @@ -477,6 +517,7 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers: "to-jelly", f, "--opt.logical-type=GRAPHS", + "--opt.jelly-version=1.1", "--to", j, ), @@ -754,10 +795,192 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers: val delimitingResponse = IoUtils.autodetectDelimiting(new FileInputStream(outFile)) delimitingResponse.isDelimited should be(true) val frames = readJellyFile(new FileInputStream(outFile)) - frames.size should be > testCardinality - for frame <- frames do frame.getRows.size should be(1) + // The stream options, and then the column batch, which is never split + frames.size should be(2) + frames(0).getRows.size should be(1) + frames(1).getColumns.getRowCount should be(testCardinality) } } + + "Jelly-RDF 1.1 text format (delimited, multiple frames)" in withFullJellyTextFile( + inFile => + withEmptyJellyFile { outFile => + RdfToJelly.runTestCommand( + List("rdf", "to-jelly", inFile, "--rows-per-frame=1", "--to", outFile), + ) + val frames = readJellyFile(new FileInputStream(outFile)) + frames.size should be > testCardinality + for frame <- frames do frame.getRows.size should be(1) + }, + rowLayout = true, + ) + } + + "choose the Jelly-RDF version and its options" when { + def outOptions(): RdfStreamOptions = + readJellyFile(ByteArrayInputStream(RdfToJelly.getOutBytes)).head.getRows.asScala.head + .getOptions + + "--opt.jelly-version=1.2 is given explicitly" in withFullJenaFile { f => + RdfToJelly.runTestCommand(List("rdf", "to-jelly", f, "--opt.jelly-version=1.2")) + outOptions().getVersion should be(JellyConstants.PROTO_VERSION_1_2_X) + } + + "--opt.jelly-version has an unknown value" in withFullJenaFile { f => + val e = intercept[ExitException] { + RdfToJelly.runTestCommand(List("rdf", "to-jelly", f, "--opt.jelly-version=2.0")) + } + e.cause.get shouldBe a[InvalidArgument] + e.cause.get.getMessage should include("--opt.jelly-version") + e.cause.get.getMessage should include("Must be one of: 1.2, 1.1") + } + + "--opt.stream-type=messages is given" in withFullJenaFile { f => + RdfToJelly.runTestCommand(List("rdf", "to-jelly", f, "--opt.stream-type=messages")) + val bytes = RdfToJelly.getOutBytes + val options = + readJellyFile(ByteArrayInputStream(bytes)).head.getRows.asScala.head.getOptions + options.getStreamType should be(RdfStreamType.MESSAGES) + // The whole input is one message + RdfSparqlTestData.readRdfMessages(bytes).map(_.size) should be(Seq(testCardinality)) + } + + "--opt.stream-type=flat is given for a PUNCTUATED Jelly-SPARQL input" in { + import RdfSparqlTestData.* + val result = sparqlJson(Seq("s", "p", "o"), Seq(statementRow("a", "p", "1"))) + val other = sparqlJson(Seq("s", "p", "o"), Seq(statementRow("b", "p", "2"))) + RdfToJelly.setStdIn( + ByteArrayInputStream(SparqlSerDesSpec.writePunctuated(Seq(result, other))), + ) + RdfToJelly.runTestCommand( + List("rdf", "to-jelly", "--in-format", "jelly-sparql", "--opt.stream-type=FLAT"), + ) + val bytes = RdfToJelly.getOutBytes + rdfStreamOptions(bytes).getStreamType should be(RdfStreamType.FLAT) + // Without message boundaries, both result sets are one graph + readRdfMessages(bytes).map(_.size) should be(Seq(2)) + } + + "--opt.stream-type has an unknown value" in withFullJenaFile { f => + val e = intercept[ExitException] { + RdfToJelly.runTestCommand(List("rdf", "to-jelly", f, "--opt.stream-type=graphs")) + } + e.cause.get shouldBe a[InvalidArgument] + e.cause.get.getMessage should include("Must be one of: flat, messages") + } + + "options of Jelly-RDF 1.2 are given for Jelly-RDF 1.1 output" in withFullJenaFile { f => + for arg <- Seq("--opt.stream-type=messages", "--opt.rdf-version=1.2") do + val e = intercept[ExitException] { + RdfToJelly.runTestCommand(List("rdf", "to-jelly", f, "--opt.jelly-version=1.1", arg)) + } + e.cause.get shouldBe a[InvalidArgument] + e.cause.get.getMessage should include("only valid in Jelly-RDF 1.2 output") + } + + "--opt.rdf-version is given" in withFullJenaFile { f => + RdfToJelly.runTestCommand(List("rdf", "to-jelly", f, "--opt.rdf-version=1.2-basic")) + outOptions().getRdfVersion should be(RdfVersion.RDF_VERSION_1_2_BASIC) + } + + "--options-from is a Jelly-RDF 1.1 file, and 1.2 is asked for" in withSpecificJellyFile( + optionsFile => + withFullJenaFile { f => + RdfToJelly.runTestCommand( + List( + "rdf", + "to-jelly", + "--options-from", + optionsFile, + "--opt.jelly-version=1.2", + "--opt.max-prefix-table-size=100", + f, + ), + ) + val from = JellyUtil.loadOptionsFromFile(optionsFile) + val opts = outOptions() + opts.getVersion should be(JellyConstants.PROTO_VERSION_1_2_X) + // Taken from the file + opts.getStreamName should be(from.getStreamName) + opts.getPhysicalType should be(from.getPhysicalType) + opts.getMaxNameTableSize should be(from.getMaxNameTableSize) + opts.getMaxDatatypeTableSize should be(from.getMaxDatatypeTableSize) + // Overridden + opts.getMaxPrefixTableSize should be(100) + // The fields of Jelly-RDF 1.1 are not taken + opts.getLogicalType should be(LogicalStreamType.UNSPECIFIED) + opts.getGeneralizedStatements should be(false) + opts.getRdfStar should be(false) + }, + fileName = "options.jelly", + ) + + "--options-from is a Jelly-RDF 1.1 GRAPHS file, and 1.2 is asked for" in withFullJenaFile { + f => + // A GRAPHS stream with 1.1 options, as the source of the options + RdfToJelly.runTestCommand( + List( + "rdf", + "to-jelly", + f, + "--opt.jelly-version=1.1", + "--opt.physical-type=GRAPHS", + "--opt.stream-name=graphs", + ), + ) + withBytesFile(RdfToJelly.getOutBytes) { optionsFile => + RdfToJelly.runTestCommand( + List("rdf", "to-jelly", "--options-from", optionsFile, "--opt.jelly-version=1.2", f), + ) + val opts = outOptions() + opts.getVersion should be(JellyConstants.PROTO_VERSION_1_2_X) + opts.getStreamName should be("graphs") + // Jelly-RDF 1.2 has no GRAPHS streams + opts.getPhysicalType should be(PhysicalStreamType.QUADS) + } + } + + "--options-from is a Jelly-RDF 1.2 file, and 1.1 is asked for" in withFullJellyFile { j => + withFullJenaFile( + f => { + RdfToJelly.runTestCommand( + List("rdf", "to-jelly", "--options-from", j, "--opt.jelly-version=1.1", f), + ) + val from = JellyUtil.loadOptionsFromFile(j) + val opts = outOptions() + JellyConstants.isRowLayout(opts.getVersion) should be(true) + opts.getStreamName should be(from.getStreamName) + opts.getMaxNameTableSize should be(from.getMaxNameTableSize) + // The defaults of Jelly-RDF 1.1, with generalized statements inferred from N-Triples + opts.getRdfStar should be(true) + opts.getGeneralizedStatements should be(true) + }, + jenaLang = RDFLanguages.NTRIPLES, + ) + } + + "--options-from is a Jelly-RDF 1.1 file, and no version is given" in withSpecificJellyFile( + optionsFile => + withFullJenaFile { f => + RdfToJelly.runTestCommand(List("rdf", "to-jelly", "--options-from", optionsFile, f)) + JellyConstants.isRowLayout(outOptions().getVersion) should be(true) + // Options of Jelly-RDF 1.2 are then refused + val e = intercept[ExitException] { + RdfToJelly.runTestCommand( + List( + "rdf", + "to-jelly", + "--options-from", + optionsFile, + "--opt.stream-type=messages", + f, + ), + ) + } + e.cause.get.getMessage should include("only valid in Jelly-RDF 1.2 output") + }, + fileName = "options.jelly", + ) } "infer stream options" when { @@ -772,6 +995,7 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers: f, "--in-format", RdfFormat.RdfProto.cliOptions.head, + "--opt.jelly-version=1.1", "--to", j, ), @@ -788,7 +1012,7 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers: testCode = { f => val (out, err) = RdfToJelly.runTestCommand( - List("rdf", "to-jelly", f, "--to", j), + List("rdf", "to-jelly", f, "--opt.jelly-version=1.1", "--to", j), ) val frames = readJellyFile(new FileInputStream(j)) val opts = frames.head.getRows.asScala.head.getOptions @@ -798,15 +1022,19 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers: ), ) "format is Jelly Text" in withEmptyJellyFile(j => - withFullJellyTextFile(testCode = { f => - val (out, err) = - RdfToJelly.runTestCommand( - List("rdf", "to-jelly", f, "--to", j), - ) - val frames = readJellyFile(new FileInputStream(j)) - val opts = frames.head.getRows.asScala.head.getOptions - opts.getGeneralizedStatements should be(true) - }), + withFullJellyTextFile( + testCode = { f => + val (out, err) = + RdfToJelly.runTestCommand( + List("rdf", "to-jelly", f, "--to", j), + ) + val frames = readJellyFile(new FileInputStream(j)) + val opts = frames.head.getRows.asScala.head.getOptions + // The options are copied from the text as they are + opts.getGeneralizedStatements should be(true) + }, + rowLayout = true, + ), ) "format is Jelly Text and options present" in withSpecificJellyFile( initialJellyFile => { @@ -848,7 +1076,9 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers: "invalid logical stream type is specified" in withFullJenaFile { f => val e = intercept[ExitException] { - RdfToJelly.runTestCommand(List("rdf", "to-jelly", f, "--opt.logical-type", "test")) + RdfToJelly.runTestCommand( + List("rdf", "to-jelly", f, "--opt.jelly-version=1.1", "--opt.logical-type", "test"), + ) } e.cause.get shouldBe a[InvalidArgument] val cause = e.cause.get.asInstanceOf[InvalidArgument] @@ -857,9 +1087,20 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers: e.code should be(1) } - "name table with size < 8 specified" in withFullJenaFile { f => + "name table with size < 128 specified" in withFullJenaFile { f => val e = intercept[ExitException] { - RdfToJelly.runTestCommand(List("rdf", "to-jelly", f, "--opt.max-name-table-size=5")) + RdfToJelly.runTestCommand(List("rdf", "to-jelly", f, "--opt.max-name-table-size=100")) + } + e.cause.get shouldBe a[InvalidArgument] + e.cause.get.getMessage should include("Must be at least 128") + e.code should be(1) + } + + "name table with size < 8 specified, in Jelly-RDF 1.1" in withFullJenaFile { f => + val e = intercept[ExitException] { + RdfToJelly.runTestCommand( + List("rdf", "to-jelly", f, "--opt.jelly-version=1.1", "--opt.max-name-table-size=5"), + ) } e.cause.get shouldBe a[JellySerializationError] val cause = e.cause.get.asInstanceOf[JellySerializationError] @@ -888,6 +1129,7 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers: "to-jelly", "--opt.logical-type=SUBJECT_GRAPHS", "--opt.physical-type=QUADS", + "--opt.jelly-version=1.1", f, ), ) @@ -910,6 +1152,7 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers: "--quiet", "--opt.logical-type=SUBJECT_GRAPHS", "--opt.physical-type=QUADS", + "--opt.jelly-version=1.1", f, ), ) @@ -1039,7 +1282,14 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers: ByteArrayInputStream(sparqlOf(writeRdf(PhysicalStreamType.QUADS, quads = quads))), ) RdfToJelly.runTestCommand( - List("rdf", "to-jelly", "--in-format", "jelly-sparql", "--opt.physical-type=GRAPHS"), + List( + "rdf", + "to-jelly", + "--in-format", + "jelly-sparql", + "--opt.physical-type=GRAPHS", + "--opt.jelly-version=1.1", + ), ) rdfStreamOptions(RdfToJelly.getOutBytes).getPhysicalType shouldBe PhysicalStreamType.GRAPHS } diff --git a/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfTranscodeSpec.scala b/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfTranscodeSpec.scala index 3f362df..da2b0e2 100644 --- a/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfTranscodeSpec.scala +++ b/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfTranscodeSpec.scala @@ -1,7 +1,8 @@ package eu.neverblink.jelly.cli.command.rdf -import eu.neverblink.jelly.cli.{ExitException, JellyTranscodingError} -import eu.neverblink.jelly.cli.command.helpers.TestFixtureHelper +import eu.neverblink.jelly.cli.* +import eu.neverblink.jelly.core.JellyConstants +import eu.neverblink.jelly.cli.command.helpers.{DataGenHelper, TestFixtureHelper} import eu.neverblink.jelly.cli.command.rdf.util.{JellyUtil, RdfJellySerializationOptions} import eu.neverblink.jelly.core.proto.v1.* import org.scalatest.matchers.should.Matchers @@ -17,15 +18,15 @@ class RdfTranscodeSpec extends AnyWordSpec, Matchers, TestFixtureHelper: private def checkOutputWithDefaultOptions(b: Array[Byte]): Unit = val outF = RdfStreamFrame.parseDelimitedFrom(ByteArrayInputStream(b)) - outF.getRows.size should be > 36 + outF.getColumns.getRowCount should be(testCardinality) val opt = outF.getRows.asScala.head.getOptions opt.getPhysicalType should be(PhysicalStreamType.TRIPLES) - opt.getLogicalType should be(LogicalStreamType.FLAT_TRIPLES) + opt.getStreamType should be(RdfStreamType.FLAT) + opt.getVersion should be(JellyConstants.PROTO_VERSION_1_2_X) opt.getMaxNameTableSize should be(defaultOpt.getMaxNameTableSize) opt.getMaxPrefixTableSize should be(defaultOpt.getMaxPrefixTableSize) opt.getMaxDatatypeTableSize should be(defaultOpt.getMaxDatatypeTableSize) - opt.getRdfStar should be(defaultOpt.getRdfStar) - opt.getGeneralizedStatements should be(defaultOpt.getGeneralizedStatements) + opt.getRdfVersion should be(RdfVersion.RDF_VERSION_UNSPECIFIED) "rdf transcode command" should { "transcode input file with no additional options" in withFullJellyFile { j => @@ -60,7 +61,7 @@ class RdfTranscodeSpec extends AnyWordSpec, Matchers, TestFixtureHelper: val outFrames = JellyUtil.iterateRdfStream(ByteArrayInputStream(outB)).toSeq outFrames.size should be(100) outFrames.foreach { f => - f.getRows.size should be >= testCardinality + f.getColumns.getRowCount should be(testCardinality) } } @@ -70,25 +71,63 @@ class RdfTranscodeSpec extends AnyWordSpec, Matchers, TestFixtureHelper: "rdf", "transcode", "--opt.max-prefix-table-size=600", - "--opt.logical-type=GRAPHS", + "--opt.rdf-version=1.2", j, ), ) val outB = RdfTranscode.getOutBytes val f = RdfStreamFrame.parseDelimitedFrom(ByteArrayInputStream(outB)) - f.getRows.size should be > testCardinality + f.getColumns.getRowCount should be(testCardinality) val opt = f.getRows.asScala.head.getOptions opt.getMaxPrefixTableSize should be(600) opt.getPhysicalType should be(PhysicalStreamType.TRIPLES) - opt.getLogicalType should be(LogicalStreamType.GRAPHS) + opt.getRdfVersion should be(RdfVersion.RDF_VERSION_1_2) } - "not allow for output name table size to smaller than the input" in withFullJellyFile { j => + "not allow for output name table size to smaller than the input" in { + // Jena's Jelly writer uses the big name table by default + RdfTranscode.setStdIn(ByteArrayInputStream(DataGenHelper.generateJellyBytes(testCardinality))) val e = intercept[ExitException] { - RdfTranscode.runTestCommand(List("rdf", "transcode", "--opt.max-name-table-size=60", j)) + RdfTranscode.runTestCommand(List("rdf", "transcode", "--opt.max-name-table-size=200")) } val cause = e.getCause cause shouldBe a[JellyTranscodingError] - cause.getMessage should include("Input lookup size cannot be greater") + cause.getMessage should include("must not be larger than the output's") + } + + "not allow for an output name table smaller than the minimum" in withFullJellyFile { j => + val e = intercept[ExitException] { + RdfTranscode.runTestCommand(List("rdf", "transcode", "--opt.max-name-table-size=60", j)) + } + e.getCause shouldBe a[InvalidArgument] + e.getCause.getMessage should include("Must be at least 128") + } + + "refuse Jelly-RDF 1.1 input, pointing to rdf to-jelly" in withFullJellyFile( + j => { + val e = intercept[ExitException] { + RdfTranscode.runTestCommand(List("rdf", "transcode", j)) + } + e.getCause shouldBe a[CriticalException] + e.getCause.getMessage should include("only supports Jelly-RDF 1.2") + e.getCause.getMessage should include("rdf to-jelly --in-format jelly") + }, + rowLayout = true, + ) + + "refuse to write Jelly-RDF 1.1" in withFullJellyFile { j => + val e = intercept[ExitException] { + RdfTranscode.runTestCommand(List("rdf", "transcode", "--opt.jelly-version=1.1", j)) + } + e.getCause shouldBe a[InvalidArgument] + e.getCause.getMessage should include("only writes Jelly-RDF 1.2") + } + + "refuse options that only exist in Jelly-RDF 1.1" in withFullJellyFile { j => + val e = intercept[ExitException] { + RdfTranscode.runTestCommand(List("rdf", "transcode", "--opt.logical-type=GRAPHS", j)) + } + e.getCause shouldBe a[InvalidArgument] + e.getCause.getMessage should include("only valid in Jelly-RDF 1.1 output") } } diff --git a/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfValidateSpec.scala b/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfValidateSpec.scala index fe80da9..9c2ee08 100644 --- a/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfValidateSpec.scala +++ b/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfValidateSpec.scala @@ -4,22 +4,121 @@ import eu.neverblink.jelly.cli.command.helpers.{RdfAdapter, TestFixtureHelper} import eu.neverblink.jelly.cli.command.helpers.RdfAdapter.* import eu.neverblink.jelly.cli.{CriticalException, ExitException} import eu.neverblink.jelly.convert.jena.JenaConverterFactory -import eu.neverblink.jelly.convert.jena.riot.JellyLanguage import eu.neverblink.jelly.core.memory.RowBuffer import eu.neverblink.jelly.core.proto.v1.* -import eu.neverblink.jelly.core.{JellyOptions, ProtoEncoder, RdfProtoDeserializationError} +import eu.neverblink.jelly.cli.util.jena.riot.JellyWriterUtil +import eu.neverblink.jelly.core.{ + JellyConstants, + JellyOptions, + ProtoEncoder, + RdfProtoDeserializationError, +} import org.apache.jena.graph.{NodeFactory, Triple} import org.apache.jena.riot.Lang import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec import java.io.{ByteArrayInputStream, ByteArrayOutputStream, FileInputStream} +import scala.annotation.nowarn import scala.util.Using import scala.jdk.CollectionConverters.* +// Some tests build Jelly-RDF 1.1 streams with the deprecated row layout encoder +@nowarn("cat=deprecation") class RdfValidateSpec extends AnyWordSpec, Matchers, TestFixtureHelper: protected val testCardinality: Int = 37 + /** For the tests that build Jelly-RDF 1.1 frames row by row. */ + private def withRowLayoutJellyFile(testCode: String => Any): Unit = + withFullJellyFile(testCode, rowLayout = true) + + /** Writes the triples as Jelly-RDF 1.1, which allows generalized statements and RDF-star. */ + private def writeRowLayout(triples: Seq[Triple]): Array[Byte] = + val out = ByteArrayOutputStream() + val writer = JellyWriterUtil.createWriter( + JellyOptions.SMALL_ALL_FEATURES.clone().setPhysicalType(PhysicalStreamType.TRIPLES) + .setVersion(JellyConstants.PROTO_VERSION_1_1_X), + 256, + false, + true, + out, + ) + triples.foreach(writer.triple) + writer.finish() + out.toByteArray + + /** Writes the triples as Jelly-RDF 1.2 with the given options. */ + private def writeColumnLayout(triples: Seq[Triple], options: RdfStreamOptions): Array[Byte] = + val out = ByteArrayOutputStream() + val writer = JellyWriterUtil.createWriter( + options.clone().setPhysicalType(PhysicalStreamType.TRIPLES), + 256, + false, + true, + out, + ) + triples.foreach(writer.triple) + writer.finish() + out.toByteArray + + private val tripleWithTripleTerm = Triple.create( + NodeFactory.createURI("http://example.org/s"), + NodeFactory.createURI("http://example.org/says"), + NodeFactory.createTripleTerm( + Triple.create( + NodeFactory.createURI("http://example.org/a"), + NodeFactory.createURI("http://example.org/b"), + NodeFactory.createLiteralString("c"), + ), + ), + ) + + "rdf validate command, for Jelly-RDF 1.2 streams" should { + "accept a stream with a triple term" in { + val jelly = writeColumnLayout(Seq(tripleWithTripleTerm), JellyOptions.SMALL_STRICT) + RdfValidate.setStdIn(ByteArrayInputStream(jelly)) + val (_, err) = RdfValidate.runTestCommand(List("rdf", "validate")) + err shouldBe empty + } + + "reject a triple term in a stream that declares RDF 1.1" in { + val frame = RdfStreamFrame.parseDelimitedFrom( + ByteArrayInputStream( + writeColumnLayout(Seq(tripleWithTripleTerm), JellyOptions.SMALL_STRICT), + ), + ) + val options = frame.getRows.asScala.head.getOptions.clone() + .setRdfVersion(RdfVersion.RDF_VERSION_1_1) + val changed = RdfStreamFrame.newInstance() + .addRows(RdfStreamRow.newInstance().setOptions(options)) + .setColumns(frame.getColumns) + RdfValidate.setStdIn(ByteArrayInputStream(changed.toByteArray)) + val e = intercept[ExitException] { + RdfValidate.runTestCommand(List("rdf", "validate")) + } + e.getMessage should include("RDF 1.1") + } + + "accept concatenated streams whose options differ in the lookup sizes" in { + val small = writeColumnLayout(Seq(tripleWithTripleTerm), JellyOptions.SMALL_STRICT) + val big = writeColumnLayout(Seq(tripleWithTripleTerm), JellyOptions.BIG_STRICT) + RdfValidate.setStdIn(ByteArrayInputStream(small ++ big)) + val (_, err) = RdfValidate.runTestCommand(List("rdf", "validate")) + err shouldBe empty + } + + "check the options against an --options-file" in withFullJellyFile { j => + RdfValidate.setStdIn(FileInputStream(j)) + RdfValidate.runTestCommand(List("rdf", "validate", "--options-file", j)) + val big = writeColumnLayout(Seq(tripleWithTripleTerm), JellyOptions.BIG_STRICT) + RdfValidate.setStdIn(ByteArrayInputStream(big)) + val e = intercept[ExitException] { + RdfValidate.runTestCommand(List("rdf", "validate", "--options-file", j)) + } + e.cause.get.getMessage should include("Stream options do not match the expected options") + } + } + "rdf validate command" should { "complain about empty input" in { val e = intercept[ExitException] { @@ -399,7 +498,7 @@ class RdfValidateSpec extends AnyWordSpec, Matchers, TestFixtureHelper: e.cause.get.getMessage should include("The version field in RdfStreamOptions is <= 0") } - "same input options supplied as in the validation source" in withFullJellyFile { j => + "same input options supplied as in the validation source" in withRowLayoutJellyFile { j => val f = rdfStreamFrame( Seq( rdfStreamRow( @@ -415,24 +514,25 @@ class RdfValidateSpec extends AnyWordSpec, Matchers, TestFixtureHelper: RdfValidate.runTestCommand(List("rdf", "validate", "--options-file", j)) } - "different input options supplied as in the validation source" in withFullJellyFile { j => - val f = rdfStreamFrame( - Seq( - rdfStreamRow( - JellyOptions.BIG_STRICT - .clone.setPhysicalType(PhysicalStreamType.TRIPLES) - .setVersion(1), + "different input options supplied as in the validation source" in withRowLayoutJellyFile { + j => + val f = rdfStreamFrame( + Seq( + rdfStreamRow( + JellyOptions.BIG_STRICT + .clone.setPhysicalType(PhysicalStreamType.TRIPLES) + .setVersion(1), + ), ), - ), - ) - RdfValidate.setStdIn(ByteArrayInputStream(f.toByteArray)) - val e = intercept[ExitException] { - RdfValidate.runTestCommand(List("rdf", "validate", "--options-file", j)) - } - e.cause.get shouldBe a[CriticalException] - e.cause.get.getMessage should include("Stream options do not match the expected options") - e.cause.get.getMessage should include("Stream options do not match the expected options") - e.cause.get.getMessage should include("""Expected: stream_name: "Stream" + ) + RdfValidate.setStdIn(ByteArrayInputStream(f.toByteArray)) + val e = intercept[ExitException] { + RdfValidate.runTestCommand(List("rdf", "validate", "--options-file", j)) + } + e.cause.get shouldBe a[CriticalException] + e.cause.get.getMessage should include("Stream options do not match the expected options") + e.cause.get.getMessage should include("Stream options do not match the expected options") + e.cause.get.getMessage should include("""Expected: stream_name: "Stream" |physical_type: PHYSICAL_STREAM_TYPE_TRIPLES |generalized_statements: true |rdf_star: true @@ -442,11 +542,13 @@ class RdfValidateSpec extends AnyWordSpec, Matchers, TestFixtureHelper: |logical_type: LOGICAL_STREAM_TYPE_FLAT_TRIPLES |version: 1""".stripMargin) - e.cause.get.getMessage should include("""Actual: physical_type: PHYSICAL_STREAM_TYPE_TRIPLES + e.cause.get.getMessage should include( + """Actual: physical_type: PHYSICAL_STREAM_TYPE_TRIPLES |max_name_table_size: 4000 |max_prefix_table_size: 150 |max_datatype_table_size: 32 - |version: 1""".stripMargin) + |version: 1""".stripMargin, + ) } } @@ -511,7 +613,7 @@ class RdfValidateSpec extends AnyWordSpec, Matchers, TestFixtureHelper: "content matches the reference RDF file, using a slice of the stream" in withFullJenaFile { jenaF => - withFullJellyFile { jellyF => + withRowLayoutJellyFile { jellyF => val frame1 = Using.resource(FileInputStream(jellyF)) { is => RdfStreamFrame.parseDelimitedFrom(is) } @@ -546,7 +648,7 @@ class RdfValidateSpec extends AnyWordSpec, Matchers, TestFixtureHelper: } "content does not match the reference RDF file, using slices" in withFullJenaFile { jenaF => - withFullJellyFile { jellyF => + withRowLayoutJellyFile { jellyF => val frame1 = Using.resource(FileInputStream(jellyF)) { is => RdfStreamFrame.parseDelimitedFrom(is) } @@ -617,19 +719,19 @@ class RdfValidateSpec extends AnyWordSpec, Matchers, TestFixtureHelper: NodeFactory.createURI("http://example.org/predicate"), NodeFactory.createBlankNode(), ) + // A triple term in the subject position is only allowed in Jelly-RDF 1.1 + val jelly = writeRowLayout(Seq(t)) withJenaFileOfContent(Seq(t), Lang.NT) { ntFile => - withJenaFileOfContent(Seq(t), JellyLanguage.JELLY) { jellyFile => - val (out, err) = RdfValidate.runTestCommand( - List( - "rdf", - "validate", - "--compare-to-rdf-file=" + ntFile, - "--compare-ordered=true", - jellyFile, - ), - ) - err shouldBe empty - } + RdfValidate.setStdIn(ByteArrayInputStream(jelly)) + val (out, err) = RdfValidate.runTestCommand( + List( + "rdf", + "validate", + "--compare-to-rdf-file=" + ntFile, + "--compare-ordered=true", + ), + ) + err shouldBe empty } } diff --git a/src/test/scala/eu/neverblink/jelly/cli/command/sparql/SparqlSerDesSpec.scala b/src/test/scala/eu/neverblink/jelly/cli/command/sparql/SparqlSerDesSpec.scala index 762ebce..a4adc92 100644 --- a/src/test/scala/eu/neverblink/jelly/cli/command/sparql/SparqlSerDesSpec.scala +++ b/src/test/scala/eu/neverblink/jelly/cli/command/sparql/SparqlSerDesSpec.scala @@ -521,10 +521,9 @@ class SparqlSerDesSpec extends AnyWordSpec with TestFixtureHelper with Matchers: sameResultSets(readResultSets(back), Seq(selectJson, askJson(false), second)) shouldBe true } - "be written by from-jelly as Jelly-RDF, one frame per result set" in { + "be written by from-jelly as Jelly-RDF, one RDF Message per result set" in { import eu.neverblink.jelly.cli.command.helpers.RdfSparqlTestData.* - import eu.neverblink.jelly.cli.command.rdf.util.JellyUtil - import eu.neverblink.jelly.core.proto.v1.{LogicalStreamType, PhysicalStreamType} + import eu.neverblink.jelly.core.proto.v1.{PhysicalStreamType, RdfStreamType} val resultSets = Seq( sparqlJson(Seq("s", "p", "o", "g"), Seq(statementRow("a", "p", "v", Some("g")))), sparqlJson(Seq("s", "p", "o"), Seq(statementRow("b", "p", "w"))), @@ -534,11 +533,10 @@ class SparqlSerDesSpec extends AnyWordSpec with TestFixtureHelper with Matchers: val rdf = SparqlFromJelly.getOutBytes val options = rdfStreamOptions(rdf) options.getPhysicalType shouldBe PhysicalStreamType.QUADS - options.getLogicalType shouldBe LogicalStreamType.DATASETS - JellyUtil.iterateRdfStream(ByteArrayInputStream(rdf)).size shouldBe 2 - readRdf(rdf) shouldBe Set( - Quad.create(iri("g"), iri("a"), iri("p"), lit("v")), - Quad.create(Quad.defaultGraphIRI, iri("b"), iri("p"), lit("w")), + options.getStreamType shouldBe RdfStreamType.MESSAGES + readRdfMessages(rdf) shouldBe Seq( + Set(Quad.create(iri("g"), iri("a"), iri("p"), lit("v"))), + Set(Quad.create(Quad.defaultGraphIRI, iri("b"), iri("p"), lit("w"))), ) } } @@ -571,7 +569,8 @@ class SparqlSerDesSpec extends AnyWordSpec with TestFixtureHelper with Matchers: } "get the RDF version from the input in to-jelly, unless --opt.rdf-version is set" in { - val rdf = writeRdf(PhysicalStreamType.TRIPLES, triples, rdfStar = false) + val rdf = + writeRdf(PhysicalStreamType.TRIPLES, triples, rdfVersion = RdfVersion.RDF_VERSION_1_1) SparqlToJelly.setStdIn(ByteArrayInputStream(rdf)) SparqlToJelly.runTestCommand( List("sparql", "to-jelly", "--in-format", "jelly-rdf", "--opt.stream-name=x"), diff --git a/src/test/scala/eu/neverblink/jelly/cli/util/jena/RdfSparqlConverterSpec.scala b/src/test/scala/eu/neverblink/jelly/cli/util/jena/RdfSparqlConverterSpec.scala index 49878b3..b49391a 100644 --- a/src/test/scala/eu/neverblink/jelly/cli/util/jena/RdfSparqlConverterSpec.scala +++ b/src/test/scala/eu/neverblink/jelly/cli/util/jena/RdfSparqlConverterSpec.scala @@ -44,13 +44,25 @@ class RdfSparqlConverterSpec extends AnyWordSpec with TestFixtureHelper with Mat /** The options the CLI uses when no --opt.* option is given. */ private def defaultRdfOptions = RdfJellySerializationOptions().asRdfStreamOptions.clone() + /** The options the CLI uses with only --opt.jelly-version=1.1. */ + private def rowLayoutRdfOptions = + RdfJellySerializationOptions(`opt.jellyVersion` = Some("1.1")).asRdfStreamOptions.clone() + private def toRdf( jelly: Array[Byte], options: RdfStreamOptions = defaultRdfOptions, delimited: Boolean = true, + streamTypeSet: Boolean = false, ): Array[Byte] = val out = ByteArrayOutputStream() - RdfSparqlConverter.sparqlToRdf(ByteArrayInputStream(jelly), out, options, 256, delimited) + RdfSparqlConverter.sparqlToRdf( + ByteArrayInputStream(jelly), + out, + options, + streamTypeSet, + 256, + delimited, + ) out.toByteArray /** Reads each frame of a Jelly-RDF stream into its own set of quads. */ @@ -65,7 +77,7 @@ class RdfSparqlConverterSpec extends AnyWordSpec with TestFixtureHelper with Mat .anyStatementDecoder(handler, JellyOptions.DEFAULT_SUPPORTED_OPTIONS) JellyUtil.iterateRdfStream(ByteArrayInputStream(jelly)).map { frame => current = Set() - frame.getRows.forEach(decoder.ingestRow) + decoder.ingestFrame(frame) current }.toSeq @@ -128,18 +140,30 @@ class RdfSparqlConverterSpec extends AnyWordSpec with TestFixtureHelper with Mat rows.flatMap(_.get("g")).toSet shouldBe Set(iri("g1"), iri("g2")) } - "announce the RDF version based on the input options by default" in { - for (tripleTerms, version) <- Seq( - false -> RdfVersion.RDF_VERSION_1_1, - true -> RdfVersion.RDF_VERSION_1_2, + "announce the RDF version of a Jelly-RDF 1.2 input by default" in { + for version <- Seq( + RdfVersion.RDF_VERSION_UNSPECIFIED, + RdfVersion.RDF_VERSION_1_1, + RdfVersion.RDF_VERSION_1_2, ) do - val jelly = toSparql(writeRdf(PhysicalStreamType.TRIPLES, triples, rdfStar = tripleTerms)) + val jelly = toSparql(writeRdf(PhysicalStreamType.TRIPLES, triples, rdfVersion = version)) val options = readFrames(jelly).head.getOptions options.getRdfVersion shouldBe version options.getMaxNameTableSize shouldBe JellySparqlOptions.BIG.getMaxNameTableSize } + "announce the RDF version based on the triple terms flag of a Jelly-RDF 1.1 input" in { + for (tripleTerms, version) <- Seq( + false -> RdfVersion.RDF_VERSION_1_1, + true -> RdfVersion.RDF_VERSION_1_2, + ) + do + val rdf = + writeRdf(PhysicalStreamType.TRIPLES, triples, rdfStar = tripleTerms, rowLayout = true) + readFrames(toSparql(rdf)).head.getOptions.getRdfVersion shouldBe version + } + "use the output options it is given" in { val jelly = toSparql( writeRdf(PhysicalStreamType.TRIPLES, triples), @@ -204,7 +228,7 @@ class RdfSparqlConverterSpec extends AnyWordSpec with TestFixtureHelper with Mat "keep the physical type it is given" in { val quadsSparql = toSparql(writeRdf(PhysicalStreamType.QUADS, quads = quads)) val graphs = - toRdf(quadsSparql, defaultRdfOptions.setPhysicalType(PhysicalStreamType.GRAPHS)) + toRdf(quadsSparql, rowLayoutRdfOptions.setPhysicalType(PhysicalStreamType.GRAPHS)) rdfStreamOptions(graphs).getPhysicalType shouldBe PhysicalStreamType.GRAPHS readRdf(graphs) shouldBe quads.toSet // 3 variables in a QUADS stream end up in the default graph @@ -248,8 +272,11 @@ class RdfSparqlConverterSpec extends AnyWordSpec with TestFixtureHelper with Mat Seq(Map("s" -> litJson("lit"), "p" -> iriJson("p"), "o" -> litJson("v"))), ) val e = intercept[CriticalException](toRdf(sparql)) - e.getMessage should include("--opt.generalized-statements=true") - val jelly = toRdf(sparql, defaultRdfOptions.setGeneralizedStatements(true)) + e.getMessage should include("Jelly-RDF 1.2 cannot store generalized statements") + e.getMessage should include("--opt.jelly-version=1.1 --opt.generalized-statements=true") + val e2 = intercept[CriticalException](toRdf(sparql, rowLayoutRdfOptions)) + e2.getMessage should include("Use --opt.generalized-statements=true") + val jelly = toRdf(sparql, rowLayoutRdfOptions.setGeneralizedStatements(true)) rdfStreamOptions(jelly).getGeneralizedStatements shouldBe true } @@ -269,8 +296,12 @@ class RdfSparqlConverterSpec extends AnyWordSpec with TestFixtureHelper with Mat val sparql = toSparql( writeRdf(PhysicalStreamType.TRIPLES, Seq(Triple.create(iri("a"), iri("says"), tripleTerm))), ) - val e = intercept[CriticalException](toRdf(sparql, defaultRdfOptions.setRdfStar(false))) - e.getMessage should include("--opt.triple-terms=true") + val e = intercept[CriticalException]( + toRdf(sparql, defaultRdfOptions.setRdfVersion(RdfVersion.RDF_VERSION_1_2_BASIC)), + ) + e.getMessage should include("--opt.rdf-version=1.2") + val e2 = intercept[CriticalException](toRdf(sparql, rowLayoutRdfOptions.setRdfStar(false))) + e2.getMessage should include("--opt.triple-terms=true") } "reject a malformed input" in { @@ -290,34 +321,67 @@ class RdfSparqlConverterSpec extends AnyWordSpec with TestFixtureHelper with Mat val abQuads = Set(quad("a", "1"), quad("b", "2")) val cdQuads = Set(quad("c", "3", Some("g")), quad("d", "4")) - "write one frame per result set, as a GRAPHS stream for 3 variables" in { + "write one RDF Message per result set" in { val jelly = toRdf(writePunctuated(Seq(ab, empty, ab), valuesPerFrame = 1)) val options = rdfStreamOptions(jelly) options.getPhysicalType shouldBe PhysicalStreamType.TRIPLES + options.getStreamType shouldBe RdfStreamType.MESSAGES + readRdfMessages(jelly) shouldBe Seq(abQuads, Set(), abQuads) + } + + "write an empty first result set as an empty message" in { + readRdfMessages(toRdf(writePunctuated(Seq(empty, ab)))) shouldBe Seq(Set(), abQuads) + } + + "write a QUADS stream when the first result set has 4 variables" in { + val jelly = toRdf(writePunctuated(Seq(cd, ab))) + val options = rdfStreamOptions(jelly) + options.getPhysicalType shouldBe PhysicalStreamType.QUADS + options.getStreamType shouldBe RdfStreamType.MESSAGES + readRdfMessages(jelly) shouldBe Seq(cdQuads, abQuads) + } + + "write all result sets as one dataset, if the FLAT stream type is set" in { + val jelly = toRdf(writePunctuated(Seq(cd, ab)), streamTypeSet = true) + rdfStreamOptions(jelly).getStreamType shouldBe RdfStreamType.FLAT + readRdf(jelly) shouldBe cdQuads ++ abQuads + } + + "write all result sets in one frame, if the output is not delimited" in { + val jelly = toRdf(writePunctuated(Seq(ab, empty, ab)), delimited = false) + RdfStreamFrame.parseFrom(jelly).getColumns.getRowCount shouldBe 4 + readRdfMessages(jelly) shouldBe Seq(abQuads, Set(), abQuads) + } + + "in Jelly-RDF 1.1, write one frame per result set, as a GRAPHS stream for 3 variables" in { + val jelly = + toRdf(writePunctuated(Seq(ab, empty, ab), valuesPerFrame = 1), rowLayoutRdfOptions) + val options = rdfStreamOptions(jelly) + options.getPhysicalType shouldBe PhysicalStreamType.TRIPLES options.getLogicalType shouldBe LogicalStreamType.GRAPHS readRdfFrames(jelly) shouldBe Seq(abQuads, Set(), abQuads) } - "write an empty first result set as a frame with only the stream options" in { - val jelly = toRdf(writePunctuated(Seq(empty, ab))) + "in Jelly-RDF 1.1, write an empty first result set as a frame with only the stream options" in { + val jelly = toRdf(writePunctuated(Seq(empty, ab)), rowLayoutRdfOptions) val first = rdfFrames(jelly).head.getRows.asScala.toSeq first.size shouldBe 1 first.head.hasOptions shouldBe true readRdfFrames(jelly) shouldBe Seq(Set(), abQuads) } - "write a DATASETS stream when the first result set has 4 variables" in { - val jelly = toRdf(writePunctuated(Seq(cd, ab))) + "in Jelly-RDF 1.1, write a DATASETS stream when the first result set has 4 variables" in { + val jelly = toRdf(writePunctuated(Seq(cd, ab)), rowLayoutRdfOptions) val options = rdfStreamOptions(jelly) options.getPhysicalType shouldBe PhysicalStreamType.QUADS options.getLogicalType shouldBe LogicalStreamType.DATASETS readRdfFrames(jelly) shouldBe Seq(cdQuads, abQuads) } - "end the graphs with each frame of a GRAPHS physical stream" in { + "in Jelly-RDF 1.1, end the graphs with each frame of a GRAPHS physical stream" in { val jelly = toRdf( writePunctuated(Seq(cd, ab, cd)), - defaultRdfOptions.setPhysicalType(PhysicalStreamType.GRAPHS), + rowLayoutRdfOptions.setPhysicalType(PhysicalStreamType.GRAPHS), ) val options = rdfStreamOptions(jelly) options.getPhysicalType shouldBe PhysicalStreamType.GRAPHS @@ -326,10 +390,10 @@ class RdfSparqlConverterSpec extends AnyWordSpec with TestFixtureHelper with Mat readRdfFrames(jelly) shouldBe Seq(cdQuads, abQuads, cdQuads) } - "keep the logical type it is given" in { + "in Jelly-RDF 1.1, keep the logical type it is given" in { val jelly = toRdf( writePunctuated(Seq(ab, ab)), - defaultRdfOptions.setLogicalType(LogicalStreamType.SUBJECT_GRAPHS), + rowLayoutRdfOptions.setLogicalType(LogicalStreamType.SUBJECT_GRAPHS), ) rdfStreamOptions(jelly).getLogicalType shouldBe LogicalStreamType.SUBJECT_GRAPHS } @@ -351,9 +415,11 @@ class RdfSparqlConverterSpec extends AnyWordSpec with TestFixtureHelper with Mat e.getMessage should include("Solution 0 of result set 1 has no value for ?p") } - "refuse to write several frames as non-delimited output" in { - toRdf(writePunctuated(Seq(ab)), delimited = false) - val e = intercept[CriticalException](toRdf(writePunctuated(Seq(ab, ab)), delimited = false)) + "in Jelly-RDF 1.1, refuse to write several frames as non-delimited output" in { + toRdf(writePunctuated(Seq(ab)), rowLayoutRdfOptions, delimited = false) + val e = intercept[CriticalException]( + toRdf(writePunctuated(Seq(ab, ab)), rowLayoutRdfOptions, delimited = false), + ) e.getMessage should include("more than one frame") } diff --git a/src/test/scala/eu/neverblink/jelly/cli/util/jena/riot/JellyMessageWriterSpec.scala b/src/test/scala/eu/neverblink/jelly/cli/util/jena/riot/JellyMessageWriterSpec.scala new file mode 100644 index 0000000..948c2af --- /dev/null +++ b/src/test/scala/eu/neverblink/jelly/cli/util/jena/riot/JellyMessageWriterSpec.scala @@ -0,0 +1,90 @@ +package eu.neverblink.jelly.cli.util.jena.riot + +import eu.neverblink.jelly.cli.CriticalException +import eu.neverblink.jelly.cli.command.helpers.RdfSparqlTestData.* +import eu.neverblink.jelly.cli.command.rdf.util.JellyUtil +import eu.neverblink.jelly.core.JellyOptions +import eu.neverblink.jelly.core.proto.v1.{PhysicalStreamType, RdfStreamType} +import org.apache.jena.graph.Triple +import org.apache.jena.sparql.core.Quad +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +import java.io.{ByteArrayInputStream, ByteArrayOutputStream} + +class JellyMessageWriterSpec extends AnyWordSpec with Matchers: + + private def options(physicalType: PhysicalStreamType) = + JellyOptions.SMALL_STRICT.clone() + .setPhysicalType(physicalType) + .setStreamType(RdfStreamType.MESSAGES) + + /** Triples with all-new IRIs, so that they fill up the name lookup. */ + private def distinctTriples(n: Int): Seq[Triple] = + (0 until n).map(i => Triple.create(iri(s"s$i"), iri(s"p$i"), iri(s"o$i"))) + + "JellyMessageWriter" should { + "write one message per endMessage call, across frames" in { + val out = ByteArrayOutputStream() + val writer = JellyMessageWriter(options(PhysicalStreamType.TRIPLES), 1, true, out) + writer.triple(triples(0)) + writer.triple(triples(1)) + writer.endMessage() + writer.endMessage() + writer.triple(triples(2)) + writer.finish() + val bytes = out.toByteArray + JellyUtil.iterateRdfStream(ByteArrayInputStream(bytes)).size should be > 1 + rdfStreamOptions(bytes).getStreamType should be(RdfStreamType.MESSAGES) + readRdfMessages(bytes) should be( + Seq( + Set(triples(0), triples(1)).map(Quad.create(Quad.defaultGraphIRI, _)), + Set(), + Set(Quad.create(Quad.defaultGraphIRI, triples(2))), + ), + ) + } + + "write quads, with the default graph" in { + val out = ByteArrayOutputStream() + val writer = JellyMessageWriter(options(PhysicalStreamType.QUADS), 256, true, out) + quads.foreach(writer.quad) + writer.finish() + readRdfMessages(out.toByteArray) should be(Seq(quads.toSet)) + } + + "refuse a quad in a TRIPLES stream" in { + val writer = JellyMessageWriter( + options(PhysicalStreamType.TRIPLES), + 256, + true, + ByteArrayOutputStream(), + ) + val e = intercept[CriticalException](writer.quad(quads.head)) + e.getMessage should include("Cannot write quads to a Jelly TRIPLES stream") + } + + "write everything in one frame if the output is not delimited" in { + val out = ByteArrayOutputStream() + val writer = JellyMessageWriter(options(PhysicalStreamType.TRIPLES), 1, false, out) + distinctTriples(10).foreach(writer.triple) + writer.endMessage() + writer.finish() + readRdfMessages(out.toByteArray).map(_.size) should be(Seq(10, 0)) + } + + "refuse non-delimited output that does not fit in one frame" in { + // The name lookup has 128 entries, so 300 triples with new IRIs need more than one frame + val writer = JellyMessageWriter( + options(PhysicalStreamType.TRIPLES), + 1, + false, + ByteArrayOutputStream(), + ) + val e = intercept[CriticalException] { + distinctTriples(300).foreach(writer.triple) + writer.finish() + } + e.getMessage should include("does not fit in a single frame") + } + }