diff --git a/README.md b/README.md index b508ee8..8218a5e 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,12 @@ The `rdf transcode` command turns one or more input Jelly streams into a single jelly-cli rdf transcode input.jelly > output.jelly ``` +If you need other options than that, use `rdf to-jelly` with Jelly input instead. It fully decodes and re-encodes the stream, so it's slower, but you can set any `--opt.*` options. The same works for Jelly-SPARQL with `sparql to-jelly --in-format jelly-sparql`. + +```shell +jelly-cli rdf to-jelly input.jelly --opt.max-name-table-size=256 > output.jelly +``` + ### Inspect Jelly files To inspect a Jelly file and get basic information describing its contents, such as stream options or number of triples in the file, run: diff --git a/build.sbt b/build.sbt index c58a54e..b002862 100644 --- a/build.sbt +++ b/build.sbt @@ -11,7 +11,7 @@ resolvers += "Sonatype OSS Snapshots" at "https://central.sonatype.com/repository/maven-snapshots" lazy val jenaV = "6.2.0" -lazy val jellyV = "3.7.3+76-b127ba1c-SNAPSHOT" +lazy val jellyV = "3.7.3+80-0f51136f-SNAPSHOT" lazy val graalvmV = "25.4.4.1.1" addCommandAlias("fixAll", "scalafixAll; scalafmtAll") diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfFromJelly.scala b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfFromJelly.scala index cbbb8a8..e1871bf 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfFromJelly.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfFromJelly.scala @@ -6,6 +6,7 @@ import eu.neverblink.jelly.cli.command.rdf.util.* import eu.neverblink.jelly.cli.command.rdf.util.RdfFormat.* import eu.neverblink.jelly.cli.util.args.IndexRange import eu.neverblink.jelly.cli.util.io.ProtoText +import eu.neverblink.jelly.cli.util.jena.riot.JellyWriterUtil import eu.neverblink.jelly.cli.util.jena.{ JenaSystemOptions, RdfSparqlConverter, @@ -107,6 +108,7 @@ object RdfFromJelly extends RdfSerDesCommand[RdfFromJellyOptions, RdfFormat.Writ format: RdfFormat.Writeable, ): Option[WriteAction] = (format, getOptions.combine) match + case (RdfFormat.JellyBinary, _) => Some(jellyToJelly) case (j: RdfFormat.Jena.StreamWriteable, _) => Some((in, out, opt) => jellyToLang(in, StreamRDFWriter.getWriterStream(out, j.jenaLang), j, opt), @@ -122,6 +124,24 @@ object RdfFromJelly extends RdfSerDesCommand[RdfFromJellyOptions, RdfFormat.Writ case (RdfFormat.JellyText, _) => Some(jellyBinaryToText) case (RdfFormat.JellySparql, _) => Some(jellyToSparql) + /** Re-encodes the Jelly-RDF stream with the default options of rdf to-jelly. Each input frame is + * written as one or more output frames. + */ + private def jellyToJelly( + inputStream: InputStream, + outputStream: OutputStream, + opt: RdfFromJellyOptions, + ): Unit = + val writer = JellyWriterUtil.createWriter( + RdfJellySerializationOptions.defaultFor(RdfFormat.JellyBinary), + RdfJellySerializationOptions.defaultRowsPerFrame, + enableNamespaceDeclarations = false, + delimited = true, + outputStream, + ) + writer.start() + jellyToLang(inputStream, writer, RdfFormat.JellyBinary, opt) + /** Converts the Jelly-RDF stream to a Jelly-SPARQL result set of ?s ?p ?o (?g), with the default * options. To set them, use sparql to-jelly --in-format jelly-rdf. */ diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJelly.scala b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJelly.scala index a31a967..32d7e57 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJelly.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJelly.scala @@ -52,9 +52,10 @@ case class RdfToJellyOptions( ) optionsFrom: Option[String] = None, @HelpMessage( - "Target number of rows per frame – the writer may slightly exceed that. Default: 256", + "Target number of rows per frame – the writer may slightly exceed that. Default: " + + RdfJellySerializationOptions.defaultRowsPerFrame, ) - rowsPerFrame: Int = 256, + rowsPerFrame: Int = RdfJellySerializationOptions.defaultRowsPerFrame, @HelpMessage( "Whether to preserve explicit namespace declarations in the output (PREFIX: in Turtle). " + "Default: false", diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/RdfFormat.scala b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/RdfFormat.scala index bd70d03..0ce61e6 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/RdfFormat.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/RdfFormat.scala @@ -94,9 +94,13 @@ object RdfFormat: override val supportsBaseIri: Boolean = true override val supportsQuads: Boolean = true - // We do not ever want to write or read from Jelly to Jelly - // So better not have it as Writeable or Readable, just mark that it's integrated into Jena - case object JellyBinary extends RdfFormat.Jena, RdfFormat.SupportsGeneralizedRdf: + /** Jelly-RDF. Reading it in rdf to-jelly and writing it in rdf from-jelly re-encodes the stream + * with new options. + */ + case object JellyBinary + extends RdfFormat.Jena.Readable, + RdfFormat.Writeable, + RdfFormat.SupportsGeneralizedRdf: override val fullName: String = "Jelly binary" override val cliOptions: List[String] = List("jelly") override val jenaLang: Lang = JellyLanguage.JELLY diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/RdfJellySerializationOptions.scala b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/RdfJellySerializationOptions.scala index c82132c..10d5a22 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/RdfJellySerializationOptions.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/RdfJellySerializationOptions.scala @@ -13,6 +13,15 @@ private val `default.opt.maxPrefixTableSize`: Int = JellyOptions.BIG_STRICT.getM private val `default.opt.maxDatatypeTableSize`: Int = JellyOptions.BIG_STRICT.getMaxDatatypeTableSize +object RdfJellySerializationOptions: + val defaultRowsPerFrame: Int = 256 + + /** The stream options rdf to-jelly would use for input in this format, with no options set. */ + def defaultFor(inputFormat: RdfFormat): RdfStreamOptions = + val options = RdfJellySerializationOptions() + options.inferGeneralized(Some(inputFormat.cliOptions.head), None) + options.asRdfStreamOptions + /** Options for serializing in Jelly-RDF */ case class RdfJellySerializationOptions( @HelpMessage("Name of the output stream (in metadata). Default: (empty)") diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlFromJelly.scala b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlFromJelly.scala index 372398b..4d448b4 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlFromJelly.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlFromJelly.scala @@ -2,7 +2,7 @@ package eu.neverblink.jelly.cli.command.sparql import caseapp.* import eu.neverblink.jelly.cli.* -import eu.neverblink.jelly.cli.command.rdf.util.RdfJellySerializationOptions +import eu.neverblink.jelly.cli.command.rdf.util.{RdfFormat, RdfJellySerializationOptions} import eu.neverblink.jelly.cli.command.sparql.util.SparqlFormat import eu.neverblink.jelly.cli.util.jena.RdfSparqlConverter @@ -67,7 +67,7 @@ object SparqlFromJelly extends SparqlSerDesCommand[SparqlFromJellyOptions]: RdfSparqlConverter.sparqlToRdf( inputStream, outputStream, - RdfJellySerializationOptions().asRdfStreamOptions, - rowsPerFrame = 256, + RdfJellySerializationOptions.defaultFor(RdfFormat.JellySparql), + RdfJellySerializationOptions.defaultRowsPerFrame, delimited = true, ) diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlSerDesCommand.scala b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlSerDesCommand.scala index 66dc729..9dab179 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlSerDesCommand.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlSerDesCommand.scala @@ -2,10 +2,23 @@ package eu.neverblink.jelly.cli.command.sparql import caseapp.* import eu.neverblink.jelly.cli.* -import eu.neverblink.jelly.cli.command.sparql.util.{SparqlFormat, SparqlJellyUtil} +import eu.neverblink.jelly.cli.command.sparql.util.{ + SparqlFormat, + SparqlJellyUtil, + SparqlResultSetReader, +} import eu.neverblink.jelly.cli.util.io.ProtoText -import eu.neverblink.jelly.convert.jena.sparql.{JenaSparqlConverterFactory, RowSetReaderJelly} -import eu.neverblink.jelly.core.proto.v1.sparql.SparqlResultsFrame +import eu.neverblink.jelly.convert.jena.sparql.{ + JellySparqlLanguage, + JenaSparqlConverterFactory, + RowSetReaderJelly, + RowSetWriterJelly, +} +import eu.neverblink.jelly.core.proto.v1.sparql.{ + SparqlResultsFrame, + SparqlResultsOptions, + SparqlStreamType, +} import org.apache.jena.riot.RIOT import org.apache.jena.riot.rowset.{RowSetReaderRegistry, RowSetWriterRegistry} import org.apache.jena.sparql.exec.QueryExecResult @@ -50,9 +63,15 @@ abstract class SparqlSerDesCommand[T <: HasJellyCommandOptions: {Parser, Help}] */ protected def jenaContext: Context = RIOT.getContext.copy() - /** Whether the Jelly-SPARQL text format should be converted to delimited Jelly-SPARQL. */ + /** Whether the Jelly-SPARQL output should be delimited. */ protected def delimitedOutput: Boolean = true + /** Stream options of the Jelly-SPARQL output, made from the options of the Jelly-SPARQL input. + * Commands that let the user set the options override this. + */ + protected def jellySparqlOutputOptions(inputOptions: SparqlResultsOptions): SparqlResultsOptions = + SparqlJellyUtil.defaultOptions(inputOptions) + /** Converts Jelly-RDF to Jelly-SPARQL. Commands that read Jelly-RDF override this. */ protected def jellyRdfToSparql(inputStream: InputStream, outputStream: OutputStream): Unit = throw CriticalException("This command cannot read Jelly-RDF.") @@ -82,6 +101,8 @@ abstract class SparqlSerDesCommand[T <: HasJellyCommandOptions: {Parser, Help}] jellyRdfToSparql(inputStream, outputStream) case (SparqlFormat.JellySparql, SparqlFormat.JellyRdf) => jellySparqlToRdf(inputStream, outputStream) + case (SparqlFormat.JellySparql, SparqlFormat.JellySparql) => + jellySparqlToSparql(inputStream, outputStream) case (f: SparqlFormat.Jena, t: SparqlFormat.Jena) => jenaConvert(f, t, inputStream, outputStream) case _ => @@ -96,26 +117,79 @@ abstract class SparqlSerDesCommand[T <: HasJellyCommandOptions: {Parser, Help}] outputStream: OutputStream, ): Unit = val context = jenaContext - val writer = RowSetWriterRegistry.getFactory(to.jenaLang).create(to.jenaLang) - def write(result: QueryExecResult): Unit = - if result.isBoolean then - writer.write(outputStream, result.booleanResult.booleanValue, context) - else writer.write(outputStream, result.rowSet, context) + // The result sets of a PUNCTUATED stream are written one after another + readResults(from, inputStream, context)(write(to, outputStream, context, _)) + + /** Reads the result sets of the input, in order. A Jelly-SPARQL stream may have many of them, + * other formats have one. + */ + protected final def readResults( + from: SparqlFormat.Jena, + inputStream: InputStream, + context: Context, + )(consumer: QueryExecResult => Unit): Unit = from match case SparqlFormat.JellySparql => - // A PUNCTUATED stream has many result sets, which are written one after another RowSetReaderJelly(RowSetReaderJelly.Options(), JenaSparqlConverterFactory.getInstance()) - .readAll(inputStream, context, write) - case _ => write(readResult(from, inputStream, context)) + .readAll(inputStream, context, consumer(_)) + case _ => + consumer( + RowSetReaderRegistry.getFactory(from.jenaLang).create(from.jenaLang) + .readAny(inputStream, context), + ) - /** Reads a result set in a format other than Jelly-SPARQL. */ - protected final def readResult( - from: SparqlFormat.Jena, - inputStream: InputStream, + /** Writes one result set, as a whole stream in the given format. */ + private def write( + to: SparqlFormat.Jena, + outputStream: OutputStream, context: Context, - ): QueryExecResult = - RowSetReaderRegistry.getFactory(from.jenaLang).create(from.jenaLang) - .readAny(inputStream, context) + result: QueryExecResult, + ): Unit = + val writer = RowSetWriterRegistry.getFactory(to.jenaLang).create(to.jenaLang) + if result.isBoolean then writer.write(outputStream, result.booleanResult.booleanValue, context) + else writer.write(outputStream, result.rowSet, context) + + /** Starts a PUNCTUATED Jelly-SPARQL stream, with the options in the context. */ + protected final def resultSetsWriter( + outputStream: OutputStream, + context: Context, + ): RowSetWriterJelly.ResultSetsWriter = + RowSetWriterJelly(RowSetWriterJelly.Options(), JenaSparqlConverterFactory.getInstance()) + .resultSetsWriter(outputStream, context) + + /** Writes the result set as the next one of a PUNCTUATED stream. */ + protected final def writeNext( + writer: RowSetWriterJelly.ResultSetsWriter, + result: QueryExecResult, + ): Unit = + if result.isBoolean then writer.write(result.booleanResult.booleanValue) + else writer.write(result.rowSet) + + /** Re-encodes a Jelly-SPARQL stream, with the options from [[jellySparqlOutputOptions]]. + * + * All result sets of a PUNCTUATED output go into one stream. A FLAT output can only have one. + */ + private def jellySparqlToSparql(inputStream: InputStream, outputStream: OutputStream): Unit = + val resultSets = SparqlResultSetReader(SparqlJellyUtil.iterateSparqlStream(inputStream)) + if !resultSets.hasNext then throw CriticalException("Empty input stream") + val options = jellySparqlOutputOptions(resultSets.options) + val context = jenaContext.set(JellySparqlLanguage.SYMBOL_STREAM_OPTIONS, options) + if options.getStreamType == SparqlStreamType.PUNCTUATED then + if !delimitedOutput then + throw InvalidArgument( + "--delimited", + "false", + Some("PUNCTUATED streams are always written delimited"), + ) + val writer = resultSetsWriter(outputStream, context) + resultSets.foreach(r => writeNext(writer, r.toQueryExecResult)) + else + write(SparqlFormat.JellySparql, outputStream, context, resultSets.next().toQueryExecResult) + if resultSets.hasNext then + throw CriticalException( + "The input has more than one result set, so it can only be written as a PUNCTUATED " + + "stream.", + ) private val frameCommentPrefix = "# Frame" diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlToJelly.scala b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlToJelly.scala index 479a300..8be2be8 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlToJelly.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlToJelly.scala @@ -5,11 +5,7 @@ import eu.neverblink.jelly.cli.* import eu.neverblink.jelly.cli.command.sparql.util.* import eu.neverblink.jelly.cli.util.jena.RdfSparqlConverter import eu.neverblink.jelly.cli.util.io.IoUtil -import eu.neverblink.jelly.convert.jena.sparql.{ - JellySparqlLanguage, - JenaSparqlConverterFactory, - RowSetWriterJelly, -} +import eu.neverblink.jelly.convert.jena.sparql.JellySparqlLanguage import eu.neverblink.jelly.core.RdfProtoDeserializationError import eu.neverblink.jelly.core.proto.v1.sparql.{SparqlResultsOptions, SparqlStreamType} import eu.neverblink.jelly.core.sparql.{JellySparqlConstants, JellySparqlOptions} @@ -130,28 +126,29 @@ object SparqlToJelly extends SparqlSerDesCommand[SparqlToJellyOptions]: getIoStreamsFromOptions(inputFiles.headOption, options.outputFile) inputFormats.head match case format: SparqlFormat.Jena if punctuated => - val writer = resultSetsWriter(outputStream) - writeResultSet(writer, format, inputStream) + val context = jenaContext + val writer = resultSetsWriter(outputStream, context) + def writeAll(format: SparqlFormat.Jena, inputStream: InputStream): Unit = + SparqlJellyUtil.translateErrors { + readResults(format, inputStream, context)(writeNext(writer, _)) + } + writeAll(format, inputStream) // The formats of the other files were checked above for case (file, format: SparqlFormat.Jena) <- inputFiles.zip(inputFormats).drop(1) do - Using.resource(IoUtil.inputStream(file))(writeResultSet(writer, format, _)) + Using.resource(IoUtil.inputStream(file))(writeAll(format, _)) outputStream.flush() case format => convert(format, SparqlFormat.JellySparql, inputStream, outputStream) - private def resultSetsWriter(outputStream: OutputStream): RowSetWriterJelly.ResultSetsWriter = - RowSetWriterJelly(RowSetWriterJelly.Options(), JenaSparqlConverterFactory.getInstance()) - .resultSetsWriter(outputStream, jenaContext) + /** The options set by the user, on top of `options-from` or the given default. */ + private def outputOptions(default: SparqlResultsOptions): SparqlResultsOptions = + getOptions.jellySerializationOptions.toSparqlResultsOptions( + Some(baseOptions.getOrElse(default)), + ) - private def writeResultSet( - writer: RowSetWriterJelly.ResultSetsWriter, - format: SparqlFormat.Jena, - inputStream: InputStream, - ): Unit = - SparqlJellyUtil.translateErrors { - val result = readResult(format, inputStream, jenaContext) - if result.isBoolean then writer.write(result.booleanResult.booleanValue) - else writer.write(result.rowSet) - } + override protected def jellySparqlOutputOptions( + inputOptions: SparqlResultsOptions, + ): SparqlResultsOptions = + outputOptions(SparqlJellyUtil.defaultOptions(inputOptions)) override protected def jellyRdfToSparql( inputStream: InputStream, @@ -160,10 +157,7 @@ object SparqlToJelly extends SparqlSerDesCommand[SparqlToJellyOptions]: RdfSparqlConverter.rdfToSparql( inputStream, outputStream, - inputOptions => - getOptions.jellySerializationOptions.toSparqlResultsOptions( - Some(baseOptions.getOrElse(RdfSparqlConverter.defaultSparqlOptions(inputOptions))), - ), + inputOptions => outputOptions(RdfSparqlConverter.defaultSparqlOptions(inputOptions)), getOptions.valuesPerFrame, getOptions.delimited, ) diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlValidate.scala b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlValidate.scala index 0d625f4..0e04db9 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlValidate.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlValidate.scala @@ -388,16 +388,9 @@ object SparqlValidate extends JellyCommand[SparqlValidateOptions]: if lang == SparqlFormat.JellySparql.jenaLang then SparqlResultSetReader(SparqlJellyUtil.iterateSparqlStream(inputStream)) else - val result = RowSetReaderRegistry.getFactory(lang).create(lang) - .readAny(inputStream, RIOT.getContext.copy()) - if result.isBoolean then Iterator(SparqlResultSet.Ask(result.booleanResult.booleanValue)) - else - val rowSet = result.rowSet - val vars = rowSet.getResultVars.asScala.toIndexedSeq - Iterator( - SparqlResultSet.Select( - vars, - Iterator.continually(rowSet).takeWhile(_.hasNext).map(_.next()) - .map(b => vars.map(b.get).toArray), - ), - ) + Iterator( + SparqlResultSet( + RowSetReaderRegistry.getFactory(lang).create(lang) + .readAny(inputStream, RIOT.getContext.copy()), + ), + ) diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlFormat.scala b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlFormat.scala index f1b498e..d7484cd 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlFormat.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlFormat.scala @@ -53,10 +53,10 @@ object SparqlFormat: override val cliOptions: List[String] = List("text") override val jenaLang: Lang = ResultSetLang.RS_Text - /** We never convert Jelly to Jelly, so this is neither Readable nor Writeable – it is only here - * so that the other side of the conversion has a name. + /** Converting Jelly-SPARQL to Jelly-SPARQL re-encodes the stream with new options. This does not + * go through Jena, see [[eu.neverblink.jelly.cli.command.sparql.SparqlSerDesCommand]]. */ - case object JellySparql extends SparqlFormat.Jena: + case object JellySparql extends SparqlFormat.Jena, SparqlFormat.Readable, SparqlFormat.Writeable: override val fullName: String = "Jelly-SPARQL" override val cliOptions: List[String] = List("jelly-sparql") override val jenaLang: Lang = JellySparqlLanguage.JELLY_SPARQL diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlJellyUtil.scala b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlJellyUtil.scala index ee9b412..31cef5b 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlJellyUtil.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlJellyUtil.scala @@ -5,7 +5,7 @@ import eu.neverblink.jelly.cli.* import eu.neverblink.jelly.cli.util.io.IoUtil import eu.neverblink.jelly.core.{RdfProtoDeserializationError, RdfProtoSerializationError} import eu.neverblink.jelly.core.proto.v1.sparql.{SparqlResultsFrame, SparqlResultsOptions} -import eu.neverblink.jelly.core.sparql.JellySparqlIoUtils +import eu.neverblink.jelly.core.sparql.{JellySparqlIoUtils, JellySparqlOptions} import org.apache.jena.riot.RiotException import java.io.InputStream @@ -37,6 +37,15 @@ object SparqlJellyUtil: // An empty input parses as an empty frame, but it's really an empty stream (false, if frame.getSerializedSize == 0 then Iterator.empty else Iterator(frame)) + /** Default options of a Jelly-SPARQL output, for the given Jelly-SPARQL input options. + * + * This is the BIG preset, with the stream type and RDF version of the input. + */ + def defaultOptions(inputOptions: SparqlResultsOptions): SparqlResultsOptions = + JellySparqlOptions.BIG.clone() + .setStreamType(inputOptions.getStreamType) + .setRdfVersion(inputOptions.getRdfVersion) + /** Reads the stream options from the first frame of a Jelly-SPARQL file. * * @throws CriticalException diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlMetricsPrinter.scala b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlMetricsPrinter.scala index e897c6c..575fabf 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlMetricsPrinter.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlMetricsPrinter.scala @@ -4,7 +4,7 @@ import com.google.protobuf.ByteString import eu.neverblink.jelly.cli.command.rdf.util.FrameInfo import eu.neverblink.jelly.cli.util.io.YamlDocBuilder import eu.neverblink.jelly.cli.util.io.YamlDocBuilder.* -import eu.neverblink.jelly.core.proto.v1.RdfLookupEntryPacked +import eu.neverblink.jelly.core.proto.v1.{RdfColumn, RdfLookupEntryPacked} import eu.neverblink.jelly.core.proto.v1.sparql.* import eu.neverblink.protoc.java.runtime.ProtoMessage @@ -43,7 +43,8 @@ final class SparqlFrameInfo( var iri: Long = 0 var bnode: Long = 0 var literal: Long = 0 - var poly: Long = 0 + var tripleTerm: Long = 0 + var layout: Long = 0 var askResult: Long = 0 var trailer: Long = 0 @@ -61,7 +62,8 @@ final class SparqlFrameInfo( this.stat.iri += other.stat.iri this.stat.bnode += other.stat.bnode this.stat.literal += other.stat.literal - this.stat.poly += other.stat.poly + this.stat.tripleTerm += other.stat.tripleTerm + this.stat.layout += other.stat.layout this.stat.askResult += other.stat.askResult this.stat.trailer += other.stat.trailer this @@ -75,13 +77,9 @@ final class SparqlFrameInfo( if isCount then entries.iterator.map(_.getValues.size.toLong).sum else measureAll(entries) - /** Measures a list of columns: the number of values in them, or their size. */ - private def measureColumns[C <: ProtoMessage[?]]( - columns: Iterable[C], - valueCount: C => Long, - ): Long = - if isCount then columns.iterator.map(valueCount).sum - else measureAll(columns) + /** The number of values of one kind in a column, or the size of the fields holding them. */ + private def measureValues(count: Int, size: => Long): Long = + if isCount then count else size /** @param resultSetStart * whether this is the first frame of a result set @@ -93,28 +91,33 @@ final class SparqlFrameInfo( stat.frame += statCollector.measure(frame) stat.row += frame.getRowCount Option(frame.getOptions).foreach(o => stat.option += statCollector.measure(o)) - stat.variable += measureAll(frame.getVariables.asScala) + stat.variable += frame.getVariables.asScala.iterator.map(statCollector.measure).sum stat.name += measureLookup(frame.getNames.asScala) stat.prefix += measureLookup(frame.getPrefixes.asScala) stat.datatype += measureLookup(frame.getDatatypes.asScala) - stat.iri += measureColumns(frame.getIriColumns.asScala, iriValues) - stat.bnode += measureColumns(frame.getBnodeColumns.asScala, bnodeValues) - stat.literal += measureColumns(frame.getLiteralColumns.asScala, literalValues) - stat.poly += measureColumns(frame.getPolyColumns.asScala, polyValues) + frame.getColumns.asScala.foreach(processColumn) Option(frame.getAskResult).foreach(r => stat.askResult += statCollector.measure(r)) Option(frame.getTrailer).foreach { t => stat.trailer += statCollector.measure(t) if t.getError.nonEmpty then trailerError = Some(t.getError) } - private def iriValues(c: SparqlIriColumn): Long = c.getNameIds.size - private def bnodeValues(c: SparqlBnodeColumn): Long = c.getValues.size - private def literalValues(c: SparqlLiteralColumn): Long = c.getLexValues.size - private def polyValues(c: SparqlPolyColumn): Long = - Option(c.getIris).map(iriValues).getOrElse(0L) + - Option(c.getBnodes).map(bnodeValues).getOrElse(0L) + - Option(c.getLiterals).map(literalValues).getOrElse(0L) + - c.getTripleTerms.size + private def processColumn(c: RdfColumn): Unit = + stat.iri += measureValues( + c.getNameIds.size, + c.getNameIds.computeUInt32SizeNoTag + c.getPrefixIds.computeUInt32SizeNoTag, + ) + stat.literal += measureValues( + c.getLexValues.size, + c.getLexValues.computeStringSizeNoTag + c.getLiteralKinds.computeUInt32SizeNoTag + + c.getLangtags.computeStringSizeNoTag + c.getLangtagDirections.computeUInt32SizeNoTag, + ) + stat.bnode += measureValues(c.getBnodes.size, c.getBnodes.computeStringSizeNoTag) + stat.tripleTerm += measureValues(c.getTripleTerms.size, measureAll(c.getTripleTerms.asScala)) + stat.layout += measureValues( + c.getLayouts.size, + c.getLayouts.computeUInt32SizeNoTag + c.getKinds.size, + ) def format(): Seq[(String, YamlValue)] = val name = statCollector.name() @@ -130,7 +133,8 @@ final class SparqlFrameInfo( "iri_value_" + name -> YamlLong(stat.iri), "bnode_value_" + name -> YamlLong(stat.bnode), "literal_value_" + name -> YamlLong(stat.literal), - "poly_value_" + name -> YamlLong(stat.poly), + "triple_term_value_" + name -> YamlLong(stat.tripleTerm), + "layout_" + name -> YamlLong(stat.layout), "ask_result_" + name -> YamlLong(stat.askResult), "trailer_" + name -> YamlLong(stat.trailer), ) ++ trailerError.map(e => "trailer_error" -> YamlString(e)) @@ -150,7 +154,7 @@ object SparqlMetricsPrinter: case Some(ask) => Seq("ask_result" -> YamlBool(ask.getValue)) case None => - val vars = frame.getVariables.asScala.map(v => YamlListElem(YamlString(v.getName))) + val vars = frame.getVariables.asScala.map(v => YamlListElem(YamlString(v))) Seq("variables" -> YamlList(vars.toSeq)) private def isPunctuated(firstFrame: SparqlResultsFrame): Boolean = diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlResultSetReader.scala b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlResultSetReader.scala index e493a15..0bea762 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlResultSetReader.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlResultSetReader.scala @@ -10,6 +10,8 @@ import eu.neverblink.jelly.core.proto.v1.sparql.{ import eu.neverblink.jelly.core.sparql.{JellySparqlOptions, SparqlResultsHandler} import org.apache.jena.graph.Node import org.apache.jena.sparql.core.Var +import org.apache.jena.sparql.engine.binding.BindingFactory +import org.apache.jena.sparql.exec.{QueryExecResult, RowSetStream} import scala.annotation.tailrec import scala.collection.mutable @@ -20,6 +22,30 @@ enum SparqlResultSet: case Select(variables: IndexedSeq[Var], rows: Iterator[Array[Node]]) case Ask(value: Boolean) + /** The result set as Jena's result. The rows are read as Jena asks for them. */ + def toQueryExecResult: QueryExecResult = this match + case Ask(value) => QueryExecResult(value) + case Select(variables, rows) => + val bindings = rows.map { row => + val builder = BindingFactory.builder() + for i <- variables.indices if row(i) != null do builder.add(variables(i), row(i)) + builder.build() + } + QueryExecResult(RowSetStream.create(variables.asJava, bindings.asJava)) + +object SparqlResultSet: + /** Jena's result as a result set. The rows are read as they are asked for. */ + def apply(result: QueryExecResult): SparqlResultSet = + if result.isBoolean then Ask(result.booleanResult.booleanValue) + else + val rowSet = result.rowSet + val variables = rowSet.getResultVars.asScala.toIndexedSeq + Select( + variables, + Iterator.continually(rowSet).takeWhile(_.hasNext).map(_.next()) + .map(b => variables.map(b.get).toArray), + ) + /** Reads the result sets of a Jelly-SPARQL stream one by one. * * Unlike Jena's reader, this one pulls frames from the stream only when it needs them, so it can @@ -57,9 +83,14 @@ final class SparqlResultSetReader( // Rows of the current result set, which must be skipped before the next one starts private var rows: Iterator[Array[Node]] = Iterator.empty + /** Stream options of the input, or null before the first frame. Known once [[hasNext]] was + * called. + */ + def options: SparqlResultsOptions = decoder.getSparqlOptions + /** Whether the stream is PUNCTUATED. Known once [[hasNext]] was called. */ def isPunctuated: Boolean = - Option(decoder.getSparqlOptions).exists(_.getStreamType == SparqlStreamType.PUNCTUATED) + Option(options).exists(_.getStreamType == SparqlStreamType.PUNCTUATED) /** Reads frames until the decoder reports something, or the stream ends. */ private def peek: Option[Event] = diff --git a/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfFromJellySpec.scala b/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfFromJellySpec.scala index 954bcb7..a70ea26 100644 --- a/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfFromJellySpec.scala +++ b/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfFromJellySpec.scala @@ -3,20 +3,28 @@ package eu.neverblink.jelly.cli.command.rdf import com.google.protobuf.InvalidProtocolBufferException import eu.neverblink.jelly.cli.* import eu.neverblink.jelly.cli.command.helpers.* -import eu.neverblink.jelly.cli.command.rdf.util.RdfFormat -import eu.neverblink.jelly.convert.jena.riot.JellyFormat +import eu.neverblink.jelly.cli.command.rdf.util.{RdfFormat, RdfJellySerializationOptions} +import eu.neverblink.jelly.convert.jena.riot.{JellyFormat, JellyLanguage} import eu.neverblink.jelly.core.proto.v1.{PhysicalStreamType, RdfStreamFrame} import eu.neverblink.jelly.core.{JellyOptions, JellyTranscoderFactory} import org.apache.jena.query.DatasetFactory import org.apache.jena.rdf.model.ModelFactory -import org.apache.jena.riot.{RDFDataMgr, RDFWriter} +import org.apache.jena.riot.system.StreamRDFLib +import org.apache.jena.riot.{RDFDataMgr, RDFParser, RDFWriter} import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import java.io.{ByteArrayInputStream, ByteArrayOutputStream, File, FileOutputStream} +import java.io.{ + ByteArrayInputStream, + ByteArrayOutputStream, + File, + FileInputStream, + FileOutputStream, +} import java.nio.file.attribute.PosixFilePermissions import java.nio.file.{Files, Paths} import scala.io.Source +import scala.jdk.CollectionConverters.* import scala.util.Using class RdfFromJellySpec extends AnyWordSpec with Matchers with TestFixtureHelper: @@ -38,6 +46,48 @@ class RdfFromJellySpec extends AnyWordSpec with Matchers with TestFixtureHelper: } "rdf from-jelly command" should { + "re-encode Jelly with the default options of rdf to-jelly" when { + "the format is given explicitly" in withFullJellyFile( + j => { + RdfFromJelly.runTestCommand(List("rdf", "from-jelly", j, "--out-format", "jelly")) + val bytes = RdfFromJelly.getOutBytes + val frames = RdfToJellySpec.readJellyFile(ByteArrayInputStream(bytes)) + val options = frames.head.getRows.asScala.head.getOptions + val expected = RdfJellySerializationOptions.defaultFor(RdfFormat.JellyBinary) + options.getMaxNameTableSize should be(expected.getMaxNameTableSize) + options.getGeneralizedStatements should be(true) + options.getRdfStar should be(true) + options.getPhysicalType should be(PhysicalStreamType.TRIPLES) + // The input has frames of 10 rows, which are kept as they are + frames.size should be > 1 + RdfToJellySpec.translateJellyBack(ByteArrayInputStream(bytes)) + .isIsomorphicWith(DataGenHelper.generateTripleModel(testCardinality)) should be(true) + }, + frameSize = 10, + ) + + "the format is inferred from the output file name" in withFullJellyFile { j => + withEmptyJellyFile { q => + RdfFromJelly.runTestCommand(List("rdf", "from-jelly", j, "--to", q)) + RdfToJellySpec.translateJellyBack(FileInputStream(q)) + .isIsomorphicWith(DataGenHelper.generateTripleModel(testCardinality)) should be(true) + } + } + + "only some frames are taken" in { + RdfFromJelly.setStdIn(ByteArrayInputStream(input10Frames)) + RdfFromJelly.runTestCommand( + List("rdf", "from-jelly", "--out-format", "jelly", "--take-frames", "3..5"), + ) + val counter = StreamRDFLib.count() + RDFParser.source(ByteArrayInputStream(RdfFromJelly.getOutBytes)) + .lang(JellyLanguage.JELLY) + .parse(counter) + // Frames 3 and 4, both the same as the others + counter.countTriples() should be(2 * testCardinality) + } + } + "handle conversion of Jelly to NTriples" when { "a file to output stream" in withFullJellyFile { j => val nQuadString = DataGenHelper.generateJenaString(testCardinality) @@ -368,31 +418,6 @@ class RdfFromJellySpec extends AnyWordSpec with Matchers with TestFixtureHelper: } } - "invalid but known output format supplied" in withFullJellyFile { j => - withEmptyJellyFile { q => - val exception = - intercept[ExitException] { - RdfFromJelly.runTestCommand( - List( - "rdf", - "from-jelly", - j, - "--to", - q, - "--out-format", - RdfFormat.JellyBinary.cliOptions.head, - ), - ) - } - val msg = InvalidFormatSpecified( - RdfFormat.JellyBinary.cliOptions.head, - RdfFromJellyPrint.validFormatsString, - ) - RdfFromJelly.getErrString should include(msg.getMessage) - exception.code should be(1) - } - } - "invalid --take-frames argument provided" in { val e = intercept[ExitException] { RdfFromJelly.runTestCommand( diff --git a/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJellySpec.scala b/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJellySpec.scala index c1c116a..8de6219 100644 --- a/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJellySpec.scala +++ b/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJellySpec.scala @@ -46,6 +46,42 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers: protected val testCardinality: Int = 33 "rdf to-jelly command" should { + "re-encode Jelly with the given options" when { + "the format is given explicitly" in withFullJellyFile { j => + RdfToJelly.runTestCommand( + List( + "rdf", + "to-jelly", + j, + "--in-format", + "jelly", + "--opt.max-name-table-size=16", + "--opt.physical-type=QUADS", + "--rows-per-frame=5", + ), + ) + val bytes = RdfToJelly.getOutBytes + val frames = readJellyFile(ByteArrayInputStream(bytes)) + val options = frames.head.getRows.asScala.head.getOptions + options.getMaxNameTableSize should be(16) + options.getPhysicalType should be(PhysicalStreamType.QUADS) + // Inferred from the input format + options.getGeneralizedStatements should be(true) + frames.size should be > 1 + translateJellyBack(ByteArrayInputStream(bytes)) + .isIsomorphicWith(DataGenHelper.generateTripleModel(testCardinality)) should be(true) + } + + "the format is inferred from the input file name" in withFullJellyFile { j => + RdfToJelly.runTestCommand(List("rdf", "to-jelly", j, "--opt.max-name-table-size=16")) + val bytes = RdfToJelly.getOutBytes + readJellyFile(ByteArrayInputStream(bytes)).head.getRows.asScala.head.getOptions + .getMaxNameTableSize should be(16) + translateJellyBack(ByteArrayInputStream(bytes)) + .isIsomorphicWith(DataGenHelper.generateTripleModel(testCardinality)) should be(true) + } + } + "handle conversion of NQuads to Jelly" when { "a file to output stream" in withFullJenaFile { f => val (out, err) = @@ -809,18 +845,6 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers: cause.format should be("invalid") } - "invalid format out of existing is specified" in withFullJenaFile { f => - val e = - intercept[ExitException] { - RdfToJelly.runTestCommand(List("rdf", "to-jelly", f, "--in-format", "jelly")) - } - e.code should be(1) - e.cause.get shouldBe a[InvalidFormatSpecified] - val cause = e.cause.get.asInstanceOf[InvalidFormatSpecified] - cause.validFormats should be(RdfToJellyPrint.validFormatsString) - cause.format should be("jelly") - } - "invalid logical stream type is specified" in withFullJenaFile { f => val e = intercept[ExitException] { diff --git a/src/test/scala/eu/neverblink/jelly/cli/command/sparql/SparqlSerDesSpec.scala b/src/test/scala/eu/neverblink/jelly/cli/command/sparql/SparqlSerDesSpec.scala index 2f5e2bd..762ebce 100644 --- a/src/test/scala/eu/neverblink/jelly/cli/command/sparql/SparqlSerDesSpec.scala +++ b/src/test/scala/eu/neverblink/jelly/cli/command/sparql/SparqlSerDesSpec.scala @@ -411,16 +411,6 @@ class SparqlSerDesSpec extends AnyWordSpec with TestFixtureHelper with Matchers: } } - "reject a format it cannot write" in { - val e = intercept[ExitException] { - SparqlFromJelly.setStdIn(ByteArrayInputStream(Array())) - SparqlFromJelly.runTestCommand( - List("sparql", "from-jelly", "--out-format", "jelly-sparql"), - ) - } - e.getCause shouldBe a[InvalidFormatSpecified] - } - "report a malformed Jelly file" in { withFile("this is definitely not Jelly", ".jellys") { f => val e = intercept[ExitException] { @@ -647,12 +637,96 @@ class SparqlSerDesSpec extends AnyWordSpec with TestFixtureHelper with Matchers: ) SparqlToJelly.getOutBytes + "Jelly-SPARQL format" should { + // Smaller than the default, so that a change of the options is visible + val smallTables = List("--opt.max-name-table-size=200") + val punctuatedInputs = + Seq(selectJson, askJson(true), selectX("""{"type":"uri","value":"http://example.org/x"}""")) + + def fromJelly(jelly: Array[Byte], args: List[String] = Nil): Array[Byte] = + SparqlFromJelly.setStdIn(ByteArrayInputStream(jelly)) + SparqlFromJelly.runTestCommand( + List("sparql", "from-jelly", "--out-format", "jelly-sparql") ++ args, + ) + SparqlFromJelly.getOutBytes + + def reEncode(jelly: Array[Byte], args: List[String]): Array[Byte] = + withBytesFile(jelly, ".jellys") { f => + SparqlToJelly.runTestCommand(List("sparql", "to-jelly", f) ++ args) + SparqlToJelly.getOutBytes + } + + "be re-encoded by from-jelly with the default options" in { + val out = fromJelly(toJelly(selectJson, ".srj", smallTables)) + val options = readFrames(out).head.getOptions + options.getMaxNameTableSize shouldBe JellySparqlOptions.BIG.getMaxNameTableSize + options.getStreamType shouldBe SparqlStreamType.FLAT + sameResultSets(readResultSets(out), Seq(selectJson)) shouldBe true + } + + "keep a PUNCTUATED stream PUNCTUATED in from-jelly" in { + val out = fromJelly(writePunctuated(punctuatedInputs)) + readFrames(out).head.getOptions.getStreamType shouldBe SparqlStreamType.PUNCTUATED + sameResultSets(readResultSets(out), punctuatedInputs) shouldBe true + } + + "be inferred from the .jellys extension in from-jelly" in { + withEmptyFile(".jellys") { target => + SparqlFromJelly.setStdIn(ByteArrayInputStream(writeJelly(selectJson))) + SparqlFromJelly.runTestCommand(List("sparql", "from-jelly", "--to", target)) + val out = Files.readAllBytes(Path.of(target)) + sameResultSets(readResultSets(out), Seq(selectJson)) shouldBe true + } + } + + "be re-encoded by to-jelly with the given options" in { + val out = reEncode(writeJelly(selectJson), smallTables) + readFrames(out).head.getOptions.getMaxNameTableSize shouldBe 200 + sameResultSets(readResultSets(out), Seq(selectJson)) shouldBe true + } + + "keep the stream type and RDF version of the input in to-jelly" in { + val input = writePunctuated(punctuatedInputs) + val out = reEncode(input, smallTables) + val options = readFrames(out).head.getOptions + options.getStreamType shouldBe SparqlStreamType.PUNCTUATED + options.getRdfVersion shouldBe readFrames(input).head.getOptions.getRdfVersion + options.getMaxNameTableSize shouldBe 200 + sameResultSets(readResultSets(out), punctuatedInputs) shouldBe true + } + + "not write several result sets as a FLAT stream in to-jelly" in { + val e = intercept[ExitException] { + reEncode(writePunctuated(punctuatedInputs), List("--opt.stream-type=flat")) + } + e.getCause shouldBe a[CriticalException] + e.getCause.getMessage should include("more than one result set") + } + + "not write a PUNCTUATED stream non-delimited in to-jelly" in { + val e = intercept[ExitException] { + reEncode(writePunctuated(punctuatedInputs), List("--delimited=false")) + } + e.getCause shouldBe a[InvalidArgument] + } + + "write all result sets of each input file in to-jelly" in { + val out = withBytesFile(writePunctuated(punctuatedInputs), ".jellys") { jellys => + withFile(selectJson, ".srj") { json => + SparqlToJelly.runTestCommand(List("sparql", "to-jelly", jellys, json)) + SparqlToJelly.getOutBytes + } + } + sameResultSets(readResultSets(out), punctuatedInputs :+ selectJson) shouldBe true + } + } + "Jelly-SPARQL text format" should { "be written by from-jelly, one commented block per frame" in { val text = toText(toJelly(selectJson, ".srj")) text should startWith("# Frame 0\n") text should not include "# Frame 1" - for expected <- Seq("options {", "variables {", "name: \"label\"", "row_count: 3") do + for expected <- Seq("options {", "variables: \"label\"", "columns {", "row_count: 3") do text should include(expected) // Non-ASCII text is written without escaping text should include("cześć")