longturtle.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377
  1. """
  2. LongTurtle RDF graph serializer for RDFLib.
  3. See http://www.w3.org/TeamSubmission/turtle/ for syntax specification.
  4. This variant, longturtle as opposed to just turtle, makes some small format changes
  5. to turtle - the original turtle serializer. It:
  6. * uses PREFIX instead of @prefix
  7. * uses BASE instead of @base
  8. * adds a new line at RDF.type, or 'a'
  9. * adds a newline and an indent for all triples with more than one object (object list)
  10. * adds a new line and ';' for the last triple in a set with '.'
  11. on the start of the next line
  12. * uses default encoding (encode()) is used instead of "latin-1"
  13. - Nicholas Car, 2023
  14. """
  15. from __future__ import annotations
  16. import warnings
  17. from typing import IO, Any, Optional
  18. from rdflib.compare import to_canonical_graph
  19. from rdflib.exceptions import Error
  20. from rdflib.graph import Graph, _TripleType
  21. from rdflib.namespace import RDF
  22. from rdflib.term import BNode, Literal, URIRef
  23. from .turtle import RecursiveSerializer
  24. __all__ = ["LongTurtleSerializer"]
  25. SUBJECT = 0
  26. VERB = 1
  27. OBJECT = 2
  28. _GEN_QNAME_FOR_DT = False
  29. _SPACIOUS_OUTPUT = False
  30. class LongTurtleSerializer(RecursiveSerializer):
  31. """LongTurtle, a Turtle serialization format.
  32. When the optional parameter `canon` is set to `True`, the graph is canonicalized
  33. before serialization. This normalizes blank node identifiers and allows for
  34. deterministic serialization of the graph. Useful when consistent outputs are required.
  35. """
  36. short_name = "longturtle"
  37. indentString = " "
  38. def __init__(self, store):
  39. self._ns_rewrite = {}
  40. self._canon = False
  41. super(LongTurtleSerializer, self).__init__(store)
  42. self.keywords = {RDF.type: "a"}
  43. self.reset()
  44. self.stream = None
  45. self._spacious: bool = _SPACIOUS_OUTPUT
  46. def addNamespace(self, prefix, namespace):
  47. # Turtle does not support prefixes that start with _
  48. # if they occur in the graph, rewrite to p_blah
  49. # this is more complicated since we need to make sure p_blah
  50. # does not already exist. And we register namespaces as we go, i.e.
  51. # we may first see a triple with prefix _9 - rewrite it to p_9
  52. # and then later find a triple with a "real" p_9 prefix
  53. # so we need to keep track of ns rewrites we made so far.
  54. if (prefix > "" and prefix[0] == "_") or self.namespaces.get(
  55. prefix, namespace
  56. ) != namespace:
  57. if prefix not in self._ns_rewrite:
  58. p = "p" + prefix
  59. while p in self.namespaces:
  60. p = "p" + p
  61. self._ns_rewrite[prefix] = p
  62. prefix = self._ns_rewrite.get(prefix, prefix)
  63. super(LongTurtleSerializer, self).addNamespace(prefix, namespace)
  64. return prefix
  65. def canonize(self):
  66. """Apply canonicalization to the store.
  67. This normalizes blank node identifiers and allows for deterministic
  68. serialization of the graph.
  69. """
  70. if not self._canon:
  71. return
  72. namespace_manager = self.store.namespace_manager
  73. store = to_canonical_graph(self.store)
  74. content = store.serialize(format="application/n-triples")
  75. lines = content.split("\n")
  76. lines.sort()
  77. graph = Graph()
  78. graph.parse(
  79. data="\n".join(lines), format="application/n-triples", skolemize=True
  80. )
  81. graph = graph.de_skolemize()
  82. graph.namespace_manager = namespace_manager
  83. self.store = graph
  84. def reset(self):
  85. super(LongTurtleSerializer, self).reset()
  86. self._shortNames = {}
  87. self._started = False
  88. self._ns_rewrite = {}
  89. self.canonize()
  90. def serialize(
  91. self,
  92. stream: IO[bytes],
  93. base: Optional[str] = None,
  94. encoding: Optional[str] = None,
  95. spacious: Optional[bool] = None,
  96. **kwargs: Any,
  97. ) -> None:
  98. self._canon = kwargs.get("canon", False)
  99. self.reset()
  100. self.stream = stream
  101. # if base is given here, use, if not and a base is set for the graph use that
  102. if base is not None:
  103. self.base = base
  104. elif self.store.base is not None:
  105. self.base = self.store.base
  106. if spacious is not None:
  107. self._spacious = spacious
  108. self.preprocess()
  109. subjects_list = self.orderSubjects()
  110. self.startDocument()
  111. firstTime = True
  112. for subject in subjects_list:
  113. if self.isDone(subject):
  114. continue
  115. if firstTime:
  116. firstTime = False
  117. if self.statement(subject) and not firstTime:
  118. self.write("\n")
  119. self.endDocument()
  120. self.base = None
  121. def preprocessTriple(self, triple: _TripleType) -> None:
  122. super(LongTurtleSerializer, self).preprocessTriple(triple)
  123. for i, node in enumerate(triple):
  124. if i == VERB:
  125. if node in self.keywords:
  126. # predicate is a keyword
  127. continue
  128. if (
  129. self.base is not None
  130. and isinstance(node, URIRef)
  131. and node.startswith(self.base)
  132. and "#" not in node.replace(self.base, "")
  133. and "/" not in node.replace(self.base, "")
  134. ):
  135. # predicate corresponds to base namespace
  136. continue
  137. # Don't use generated prefixes for subjects and objects
  138. self.get_pname(node, gen_prefix=(i == VERB))
  139. if isinstance(node, Literal) and node.datatype:
  140. self.get_pname(node.datatype, gen_prefix=_GEN_QNAME_FOR_DT)
  141. p = triple[1]
  142. if isinstance(p, BNode): # hmm - when is P ever a bnode?
  143. self._references[p] += 1
  144. def get_pname(self, uri, gen_prefix=True):
  145. if not isinstance(uri, URIRef):
  146. return None
  147. try:
  148. parts = self.store.compute_qname(uri, generate=gen_prefix)
  149. except Exception:
  150. # is the uri a namespace in itself?
  151. pfx = self.store.store.prefix(uri)
  152. if pfx is not None:
  153. parts = (pfx, uri, "")
  154. else:
  155. # nothing worked
  156. return None
  157. prefix, namespace, local = parts
  158. # To understand treatment of % character refer to Productions for terminal PLX at
  159. # https://www.w3.org/TR/turtle/#grammar-production-PLX
  160. # Only % NOT followed by two hex chars requires manual backslash escaping
  161. local = local.replace(r"(", r"\(").replace(r")", r"\)")
  162. local = self.LOCALNAME_PECRENT_CHARACTER_REQUIRING_ESCAPE_REGEX.sub(
  163. "\\%", local
  164. )
  165. # PName cannot end with .
  166. if local.endswith("."):
  167. return None
  168. prefix = self.addNamespace(prefix, namespace)
  169. return "%s:%s" % (prefix, local)
  170. def getQName(self, uri, gen_prefix=True):
  171. warnings.warn(
  172. "LongTurtleSerializer.getQName is deprecated, use LongTurtleSerializer.get_pname instead.",
  173. DeprecationWarning,
  174. stacklevel=2,
  175. )
  176. return self.get_pname(uri, gen_prefix)
  177. def startDocument(self):
  178. self._started = True
  179. ns_list = sorted(self.namespaces.items())
  180. if self.base:
  181. self.write(self.indent() + "BASE <%s>\n" % self.base)
  182. for prefix, uri in ns_list:
  183. self.write(self.indent() + "PREFIX %s: <%s>\n" % (prefix, uri))
  184. if ns_list and self._spacious:
  185. self.write("\n")
  186. def endDocument(self):
  187. if self._spacious:
  188. self.write("\n")
  189. def statement(self, subject):
  190. self.subjectDone(subject)
  191. return self.s_squared(subject) or self.s_default(subject)
  192. def s_default(self, subject):
  193. self.write("\n" + self.indent())
  194. self.path(subject, SUBJECT)
  195. self.write("\n" + self.indent())
  196. self.predicateList(subject)
  197. self.write("\n.")
  198. return True
  199. def s_squared(self, subject):
  200. if (self._references[subject] > 0) or not isinstance(subject, BNode):
  201. return False
  202. self.write("\n" + self.indent() + "[]")
  203. self.predicateList(subject, newline=False)
  204. self.write("\n.")
  205. return True
  206. def path(self, node, position, newline=False):
  207. if not (
  208. self.p_squared(node, position) or self.p_default(node, position, newline)
  209. ):
  210. raise Error("Cannot serialize node '%s'" % (node,))
  211. def p_default(self, node, position, newline=False):
  212. if position != SUBJECT and not newline:
  213. self.write(" ")
  214. self.write(self.label(node, position))
  215. return True
  216. def label(self, node, position):
  217. if node == RDF.nil:
  218. return "()"
  219. if position is VERB and node in self.keywords:
  220. return self.keywords[node]
  221. if isinstance(node, Literal):
  222. return node._literal_n3(
  223. use_plain=True,
  224. qname_callback=lambda dt: self.get_pname(dt, _GEN_QNAME_FOR_DT),
  225. )
  226. else:
  227. node = self.relativize(node)
  228. return self.get_pname(node, position == VERB) or node.n3()
  229. def p_squared(
  230. self,
  231. node,
  232. position,
  233. ):
  234. if (
  235. not isinstance(node, BNode)
  236. or node in self._serialized
  237. or self._references[node] > 1
  238. or position == SUBJECT
  239. ):
  240. return False
  241. if self.isValidList(node):
  242. # this is a list
  243. self.depth += 2
  244. self.write(" (\n")
  245. self.depth -= 2
  246. self.doList(node)
  247. self.write("\n" + self.indent() + ")")
  248. else:
  249. # this is a Blank Node
  250. self.subjectDone(node)
  251. self.write("\n" + self.indent(1) + "[\n")
  252. self.depth += 1
  253. self.predicateList(node)
  254. self.depth -= 1
  255. self.write("\n" + self.indent(1) + "]")
  256. return True
  257. def isValidList(self, l_):
  258. """
  259. Checks if l is a valid RDF list, i.e. no nodes have other properties.
  260. """
  261. try:
  262. if self.store.value(l_, RDF.first) is None:
  263. return False
  264. except Exception:
  265. return False
  266. while l_:
  267. if l_ != RDF.nil and len(list(self.store.predicate_objects(l_))) != 2:
  268. return False
  269. l_ = self.store.value(l_, RDF.rest)
  270. return True
  271. def doList(self, l_):
  272. i = 0
  273. while l_:
  274. item = self.store.value(l_, RDF.first)
  275. if item is not None:
  276. if i == 0:
  277. self.write(self.indent(1))
  278. else:
  279. self.write("\n" + self.indent(1))
  280. self.path(item, OBJECT, newline=True)
  281. self.subjectDone(l_)
  282. l_ = self.store.value(l_, RDF.rest)
  283. i += 1
  284. def predicateList(self, subject, newline=False):
  285. properties = self.buildPredicateHash(subject)
  286. propList = self.sortProperties(properties)
  287. if len(propList) == 0:
  288. return
  289. self.write(self.indent(1))
  290. self.verb(propList[0], newline=True)
  291. self.objectList(properties[propList[0]])
  292. for predicate in propList[1:]:
  293. self.write(" ;\n" + self.indent(1))
  294. self.verb(predicate, newline=True)
  295. self.objectList(properties[predicate])
  296. self.write(" ;")
  297. def verb(self, node, newline=False):
  298. self.path(node, VERB, newline)
  299. def objectList(self, objects):
  300. count = len(objects)
  301. if count == 0:
  302. return
  303. depthmod = (count == 1) and 0 or 1
  304. self.depth += depthmod
  305. first_nl = False
  306. if count > 1:
  307. if not isinstance(objects[0], BNode):
  308. self.write("\n" + self.indent(1))
  309. else:
  310. self.write(" ")
  311. first_nl = True
  312. self.path(objects[0], OBJECT, newline=first_nl)
  313. for obj in objects[1:]:
  314. self.write(" ,")
  315. if not isinstance(obj, BNode):
  316. self.write("\n" + self.indent(1))
  317. self.path(obj, OBJECT, newline=True)
  318. self.depth -= depthmod