sparqlstore.py 38 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997
  1. """
  2. This is an RDFLib store around Ivan Herman et al.'s SPARQL service wrapper.
  3. This was first done in layer-cake, and then ported to RDFLib
  4. """
  5. from __future__ import annotations
  6. import collections
  7. import re
  8. from typing import (
  9. TYPE_CHECKING,
  10. Any,
  11. Callable,
  12. Dict,
  13. Generator,
  14. Iterable,
  15. Iterator,
  16. List,
  17. Mapping,
  18. Optional,
  19. Tuple,
  20. Union,
  21. overload,
  22. )
  23. from rdflib.graph import DATASET_DEFAULT_GRAPH_ID, Graph
  24. from rdflib.plugins.stores.regexmatching import NATIVE_REGEX
  25. from rdflib.store import Store
  26. from rdflib.term import BNode, Identifier, Node, URIRef, Variable
  27. if TYPE_CHECKING:
  28. import typing_extensions as te # noqa: I001
  29. from rdflib.graph import (
  30. _TripleType,
  31. _ContextType,
  32. _QuadType,
  33. _TripleChoiceType,
  34. _TriplePatternType,
  35. _SubjectType,
  36. _PredicateType,
  37. _ObjectType,
  38. _ContextIdentifierType,
  39. )
  40. from rdflib.plugins.sparql.sparql import Query, Update
  41. from rdflib.query import Result, ResultRow
  42. from .sparqlconnector import SPARQLConnector
  43. # Defines some SPARQL keywords
  44. LIMIT = "LIMIT"
  45. OFFSET = "OFFSET"
  46. ORDERBY = "ORDER BY"
  47. BNODE_IDENT_PATTERN = re.compile(r"(?P<label>_\:[^\s]+)")
  48. _NodeToSparql = Callable[["Node"], str]
  49. def _node_to_sparql(node: Node) -> str:
  50. if isinstance(node, BNode):
  51. raise Exception(
  52. "SPARQLStore does not support BNodes! "
  53. "See http://www.w3.org/TR/sparql11-query/#BGPsparqlBNodes"
  54. )
  55. return node.n3()
  56. class SPARQLStore(SPARQLConnector, Store):
  57. """An RDFLib store around a SPARQL endpoint
  58. This is context-aware and should work as expected
  59. when a context is specified.
  60. For ConjunctiveGraphs, reading is done from the "default graph". Exactly
  61. what this means depends on your endpoint, because SPARQL does not offer a
  62. simple way to query the union of all graphs as it would be expected for a
  63. ConjuntiveGraph. This is why we recommend using Dataset instead, which is
  64. motivated by the SPARQL 1.1.
  65. Fuseki/TDB has a flag for specifying that the default graph
  66. is the union of all graphs (`tdb:unionDefaultGraph` in the Fuseki config).
  67. !!! warning "Blank nodes
  68. By default the SPARQL Store does not support blank-nodes!
  69. As blank-nodes act as variables in SPARQL queries,
  70. there is no way to query for a particular blank node without
  71. using non-standard SPARQL extensions.
  72. See http://www.w3.org/TR/sparql11-query/#BGPsparqlBNodes
  73. You can make use of such extensions through the `node_to_sparql`
  74. argument. For example if you want to transform BNode('0001') into
  75. "<bnode:b0001>", you can use a function like this:
  76. ```python
  77. >>> def my_bnode_ext(node):
  78. ... if isinstance(node, BNode):
  79. ... return '<bnode:b%s>' % node
  80. ... return _node_to_sparql(node)
  81. >>> store = SPARQLStore('http://dbpedia.org/sparql',
  82. ... node_to_sparql=my_bnode_ext)
  83. ```
  84. You can request a particular result serialization with the
  85. `returnFormat` parameter. This is a string that must have a
  86. matching plugin registered. Built in is support for `xml`,
  87. `json`, `csv`, `tsv` and `application/rdf+xml`.
  88. The underlying SPARQLConnector uses the urllib library.
  89. Any extra kwargs passed to the SPARQLStore connector are passed to
  90. urllib when doing HTTP calls. I.e. you have full control of
  91. cookies/auth/headers.
  92. Form example:
  93. ```python
  94. >>> store = SPARQLStore('...my endpoint ...', auth=('user','pass'))
  95. ```
  96. will use HTTP basic auth.
  97. """
  98. formula_aware = False
  99. transaction_aware = False
  100. graph_aware = True
  101. regex_matching = NATIVE_REGEX
  102. def __init__(
  103. self,
  104. query_endpoint: Optional[str] = None,
  105. sparql11: bool = True,
  106. context_aware: bool = True,
  107. node_to_sparql: _NodeToSparql = _node_to_sparql,
  108. returnFormat: Optional[str] = "xml", # noqa: N803
  109. auth: Optional[Tuple[str, str]] = None,
  110. **sparqlconnector_kwargs,
  111. ):
  112. super(SPARQLStore, self).__init__(
  113. query_endpoint=query_endpoint,
  114. returnFormat=returnFormat,
  115. auth=auth,
  116. **sparqlconnector_kwargs,
  117. )
  118. self.node_to_sparql = node_to_sparql
  119. self.nsBindings: Dict[str, Any] = {}
  120. self.sparql11 = sparql11
  121. self.context_aware = context_aware
  122. self.graph_aware = context_aware
  123. self._queries = 0
  124. # type error: Missing return statement
  125. def open(self, configuration: Union[str, tuple[str, str]], create: bool = False) -> Optional[int]: # type: ignore[return]
  126. """This method is included so that calls to this Store via Graph, e.g. Graph("SPARQLStore"),
  127. can set the required parameters
  128. """
  129. if type(configuration) is str:
  130. self.query_endpoint = configuration
  131. else:
  132. raise Exception(
  133. "configuration must be a string (a single query endpoint URI)"
  134. )
  135. # Database Management Methods
  136. def create(self, configuration: str) -> None:
  137. raise TypeError(
  138. "The SPARQL Store is read only. Try SPARQLUpdateStore for read/write."
  139. )
  140. def destroy(self, configuration: str) -> None:
  141. raise TypeError("The SPARQL store is read only")
  142. # Transactional interfaces
  143. def commit(self) -> None:
  144. raise TypeError("The SPARQL store is read only")
  145. def rollback(self) -> None:
  146. raise TypeError("The SPARQL store is read only")
  147. def add(
  148. self, _: _TripleType, context: _ContextType = None, quoted: bool = False
  149. ) -> None:
  150. raise TypeError("The SPARQL store is read only")
  151. def addN(self, quads: Iterable[_QuadType]) -> None: # noqa: N802
  152. raise TypeError("The SPARQL store is read only")
  153. # type error: Signature of "remove" incompatible with supertype "Store"
  154. def remove( # type: ignore[override]
  155. self, _: _TriplePatternType, context: Optional[_ContextType]
  156. ) -> None:
  157. raise TypeError("The SPARQL store is read only")
  158. # type error: Signature of "update" incompatible with supertype "SPARQLConnector"
  159. def update( # type: ignore[override]
  160. self,
  161. query: Union[Update, str],
  162. initNs: Dict[str, Any] = {}, # noqa: N803
  163. initBindings: Dict[str, Identifier] = {}, # noqa: N803
  164. queryGraph: Identifier = None, # noqa: N803
  165. DEBUG: bool = False, # noqa: N803
  166. ) -> None:
  167. raise TypeError("The SPARQL store is read only")
  168. def _query(self, *args: Any, **kwargs: Any) -> Result:
  169. self._queries += 1
  170. return super(SPARQLStore, self).query(*args, **kwargs)
  171. def _inject_prefixes(self, query: str, extra_bindings: Mapping[str, Any]) -> str:
  172. bindings = set(list(self.nsBindings.items()) + list(extra_bindings.items()))
  173. if not bindings:
  174. return query
  175. return "\n".join(
  176. [
  177. "\n".join(["PREFIX %s: <%s>" % (k, v) for k, v in bindings]),
  178. "", # separate ns_bindings from query with an empty line
  179. query,
  180. ]
  181. )
  182. # type error: Signature of "query" incompatible with supertype "SPARQLConnector"
  183. # type error: Signature of "query" incompatible with supertype "Store"
  184. def query( # type: ignore[override]
  185. self,
  186. query: Union[Query, str],
  187. initNs: Optional[Mapping[str, Any]] = None, # noqa: N803
  188. initBindings: Optional[Mapping[str, Identifier]] = None, # noqa: N803
  189. queryGraph: Optional[str] = None, # noqa: N803
  190. DEBUG: bool = False, # noqa: N803
  191. ) -> Result:
  192. self.debug = DEBUG
  193. assert isinstance(query, str)
  194. if initNs is not None and len(initNs) > 0:
  195. query = self._inject_prefixes(query, initNs)
  196. if initBindings:
  197. if not self.sparql11:
  198. raise Exception("initBindings not supported for SPARQL 1.0 Endpoints.")
  199. v = list(initBindings)
  200. # VALUES was added to SPARQL 1.1 on 2012/07/24
  201. query += "\nVALUES ( %s )\n{ ( %s ) }\n" % (
  202. " ".join("?" + str(x) for x in v),
  203. " ".join(self.node_to_sparql(initBindings[x]) for x in v),
  204. )
  205. return self._query(
  206. query, default_graph=queryGraph if self._is_contextual(queryGraph) else None
  207. )
  208. # type error: Return type "Iterator[Tuple[Tuple[Node, Node, Node], None]]" of "triples" incompatible with return type "Iterator[Tuple[Tuple[Node, Node, Node], Iterator[Optional[Graph]]]]"
  209. def triples( # type: ignore[override]
  210. self, spo: _TriplePatternType, context: Optional[_ContextType] = None
  211. ) -> Iterator[Tuple[_TripleType, None]]:
  212. """
  213. - tuple **(s, o, p)**
  214. the triple used as filter for the SPARQL select.
  215. (None, None, None) means anything.
  216. - context **context**
  217. the graph effectively calling this method.
  218. Returns a tuple of triples executing essentially a SPARQL like
  219. SELECT ?subj ?pred ?obj WHERE { ?subj ?pred ?obj }
  220. **context** may include three parameter
  221. to refine the underlying query:
  222. * LIMIT: an integer to limit the number of results
  223. * OFFSET: an integer to enable paging of results
  224. * ORDERBY: an instance of Variable('s'), Variable('o') or Variable('p') or, by default, the first 'None' from the given triple
  225. !!! warning "Limit and offset
  226. - Using LIMIT or OFFSET automatically include ORDERBY otherwise this is
  227. because the results are retrieved in a not deterministic way (depends on
  228. the walking path on the graph)
  229. - Using OFFSET without defining LIMIT will discard the first OFFSET - 1 results
  230. ```python
  231. a_graph.LIMIT = limit
  232. a_graph.OFFSET = offset
  233. triple_generator = a_graph.triples(mytriple):
  234. # do something
  235. # Removes LIMIT and OFFSET if not required for the next triple() calls
  236. del a_graph.LIMIT
  237. del a_graph.OFFSET
  238. ```
  239. """
  240. s, p, o = spo
  241. vars = []
  242. if not s:
  243. s = Variable("s")
  244. vars.append(s)
  245. if not p:
  246. p = Variable("p")
  247. vars.append(p)
  248. if not o:
  249. o = Variable("o")
  250. vars.append(o)
  251. if vars:
  252. v = " ".join([term.n3() for term in vars])
  253. verb = "SELECT %s " % v
  254. else:
  255. verb = "ASK"
  256. nts = self.node_to_sparql
  257. query = "%s { %s %s %s }" % (verb, nts(s), nts(p), nts(o))
  258. # The ORDER BY is necessary
  259. if (
  260. hasattr(context, LIMIT)
  261. or hasattr(context, OFFSET)
  262. or hasattr(context, ORDERBY)
  263. ):
  264. var = None
  265. if isinstance(s, Variable):
  266. var = s
  267. elif isinstance(p, Variable):
  268. var = p
  269. elif isinstance(o, Variable):
  270. var = o
  271. elif hasattr(context, ORDERBY) and isinstance(
  272. getattr(context, ORDERBY), Variable
  273. ):
  274. var = getattr(context, ORDERBY)
  275. # type error: Item "None" of "Optional[Variable]" has no attribute "n3"
  276. query = query + " %s %s" % (ORDERBY, var.n3()) # type: ignore[union-attr]
  277. try:
  278. query = query + " LIMIT %s" % int(getattr(context, LIMIT))
  279. except (ValueError, TypeError, AttributeError):
  280. pass
  281. try:
  282. query = query + " OFFSET %s" % int(getattr(context, OFFSET))
  283. except (ValueError, TypeError, AttributeError):
  284. pass
  285. result = self._query(
  286. query,
  287. # type error: Item "None" of "Optional[Graph]" has no attribute "identifier"
  288. default_graph=context.identifier if self._is_contextual(context) else None, # type: ignore[union-attr]
  289. )
  290. if vars:
  291. if type(result) is tuple:
  292. if result[0] == 401:
  293. raise ValueError(
  294. "It looks like you need to authenticate with this SPARQL Store. HTTP unauthorized"
  295. )
  296. for row in result:
  297. if TYPE_CHECKING:
  298. # This will be a ResultRow because if vars is truthish then
  299. # the query will be a SELECT query.
  300. assert isinstance(row, ResultRow)
  301. yield (
  302. # type error: No overload variant of "get" of "ResultRow" matches argument types "Node", "Node"
  303. row.get(s, s), # type: ignore[call-overload]
  304. row.get(p, p), # type: ignore[call-overload]
  305. row.get(o, o), # type: ignore[call-overload]
  306. ), None # why is the context here not the passed in graph 'context'?
  307. else:
  308. if result.askAnswer:
  309. yield (s, p, o), None
  310. def triples_choices(
  311. self,
  312. _: _TripleChoiceType,
  313. context: Optional[_ContextType] = None,
  314. ) -> Generator[
  315. Tuple[
  316. Tuple[_SubjectType, _PredicateType, _ObjectType],
  317. Iterator[Optional[_ContextType]],
  318. ],
  319. None,
  320. None,
  321. ]:
  322. """
  323. A variant of triples that can take a list of terms instead of a
  324. single term in any slot. Stores can implement this to optimize
  325. the response time from the import default 'fallback' implementation,
  326. which will iterate over each term in the list and dispatch to
  327. triples.
  328. """
  329. raise NotImplementedError("Triples choices currently not supported")
  330. def __len__(self, context: Optional[_ContextType] = None) -> int:
  331. if not self.sparql11:
  332. raise NotImplementedError(
  333. "For performance reasons, this is not"
  334. + "supported for sparql1.0 endpoints"
  335. )
  336. else:
  337. q = "SELECT (count(*) as ?c) WHERE {?s ?p ?o .}"
  338. result = self._query(
  339. q,
  340. # type error: Item "None" of "Optional[Graph]" has no attribute "identifier"
  341. default_graph=(
  342. context.identifier # type: ignore[union-attr]
  343. if self._is_contextual(context)
  344. else None
  345. ),
  346. )
  347. # type error: Item "Tuple[Node, ...]" of "Union[Tuple[Node, Node, Node], bool, ResultRow]" has no attribute "c"
  348. return int(next(iter(result)).c) # type: ignore[union-attr]
  349. # type error: Return type "Generator[Identifier, None, None]" of "contexts" incompatible with return type "Generator[Graph, None, None]" in supertype "Store"
  350. def contexts( # type: ignore[override]
  351. self, triple: Optional[_TripleType] = None
  352. ) -> Generator[_ContextIdentifierType, None, None]:
  353. """
  354. Iterates over results to `SELECT ?NAME { GRAPH ?NAME { ?s ?p ?o } }`
  355. or `SELECT ?NAME { GRAPH ?NAME {} }` if triple is `None`.
  356. Returns instances of this store with the SPARQL wrapper
  357. object updated via addNamedGraph(?NAME).
  358. This causes a named-graph-uri key / value pair to be sent over
  359. the protocol.
  360. Please note that some SPARQL endpoints are not able to find empty named
  361. graphs.
  362. """
  363. if triple:
  364. nts = self.node_to_sparql
  365. s, p, o = triple
  366. params = (
  367. nts(s if s else Variable("s")),
  368. nts(p if p else Variable("p")),
  369. nts(o if o else Variable("o")),
  370. )
  371. q = "SELECT ?name WHERE { GRAPH ?name { %s %s %s }}" % params
  372. else:
  373. q = "SELECT ?name WHERE { GRAPH ?name {} }"
  374. result = self._query(q)
  375. # type error: Item "bool" of "Union[Tuple[Node, Node, Node], bool, ResultRow]" has no attribute "name"
  376. # error: Generator has incompatible item type "Union[Any, Identifier]"; expected "IdentifiedNode"
  377. return (row.name for row in result) # type: ignore[union-attr,misc]
  378. # Namespace persistence interface implementation
  379. def bind(self, prefix: str, namespace: URIRef, override: bool = True) -> None:
  380. bound_prefix = self.prefix(namespace)
  381. if override and bound_prefix:
  382. del self.nsBindings[bound_prefix]
  383. self.nsBindings[prefix] = namespace
  384. def prefix(self, namespace: URIRef) -> Optional[str]:
  385. """ """
  386. return dict([(v, k) for k, v in self.nsBindings.items()]).get(namespace)
  387. def namespace(self, prefix: str) -> Optional[URIRef]:
  388. return self.nsBindings.get(prefix)
  389. def namespaces(self) -> Iterator[Tuple[str, URIRef]]:
  390. for prefix, ns in self.nsBindings.items():
  391. yield prefix, ns
  392. def add_graph(self, graph: Graph) -> None:
  393. raise TypeError("The SPARQL store is read only")
  394. def remove_graph(self, graph: Graph) -> None:
  395. raise TypeError("The SPARQL store is read only")
  396. @overload
  397. def _is_contextual(self, graph: None) -> te.Literal[False]: ...
  398. @overload
  399. def _is_contextual(self, graph: Optional[Union[Graph, str]]) -> bool: ...
  400. def _is_contextual(self, graph: Optional[Union[Graph, str]]) -> bool:
  401. """Returns `True` if the "GRAPH" keyword must appear
  402. in the final SPARQL query sent to the endpoint.
  403. """
  404. if (not self.context_aware) or (graph is None):
  405. return False
  406. if isinstance(graph, str):
  407. return graph != "__UNION__"
  408. else:
  409. return graph.identifier != DATASET_DEFAULT_GRAPH_ID
  410. def subjects(
  411. self,
  412. predicate: Optional[_PredicateType] = None,
  413. object: Optional[_ObjectType] = None,
  414. ) -> Generator[_SubjectType, None, None]:
  415. """A generator of subjects with the given predicate and object"""
  416. for t, c in self.triples((None, predicate, object)):
  417. yield t[0]
  418. def predicates(
  419. self,
  420. subject: Optional[_SubjectType] = None,
  421. object: Optional[_ObjectType] = None,
  422. ) -> Generator[_PredicateType, None, None]:
  423. """A generator of predicates with the given subject and object"""
  424. for t, c in self.triples((subject, None, object)):
  425. yield t[1]
  426. def objects(
  427. self,
  428. subject: Optional[_SubjectType] = None,
  429. predicate: Optional[_PredicateType] = None,
  430. ) -> Generator[_ObjectType, None, None]:
  431. """A generator of objects with the given subject and predicate"""
  432. for t, c in self.triples((subject, predicate, None)):
  433. yield t[2]
  434. def subject_predicates(
  435. self, object: Optional[_ObjectType] = None
  436. ) -> Generator[Tuple[_SubjectType, _PredicateType], None, None]:
  437. """A generator of (subject, predicate) tuples for the given object"""
  438. for t, c in self.triples((None, None, object)):
  439. yield t[0], t[1]
  440. def subject_objects(
  441. self, predicate: Optional[_PredicateType] = None
  442. ) -> Generator[Tuple[_SubjectType, _ObjectType], None, None]:
  443. """A generator of (subject, object) tuples for the given predicate"""
  444. for t, c in self.triples((None, predicate, None)):
  445. yield t[0], t[2]
  446. def predicate_objects(
  447. self, subject: Optional[_SubjectType] = None
  448. ) -> Generator[Tuple[_PredicateType, _ObjectType], None, None]:
  449. """A generator of (predicate, object) tuples for the given subject"""
  450. for t, c in self.triples((subject, None, None)):
  451. yield t[1], t[2]
  452. class SPARQLUpdateStore(SPARQLStore):
  453. """A store using SPARQL queries for reading and SPARQL Update for changes.
  454. This can be context-aware, if so, any changes will be to the given named
  455. graph only.
  456. In favor of the SPARQL 1.1 motivated Dataset, we advise against using this
  457. with ConjunctiveGraphs, as it reads and writes from and to the
  458. "default graph". Exactly what this means depends on the endpoint and can
  459. result in confusion.
  460. For Graph objects, everything works as expected.
  461. See the [`SPARQLStore`][rdflib.plugins.stores.sparqlstore.SPARQLStore] base class for more information.
  462. """
  463. where_pattern = re.compile(r"""(?P<where>WHERE\s*\{)""", re.IGNORECASE)
  464. ##############################################################
  465. # Regex for injecting GRAPH blocks into updates on a context #
  466. ##############################################################
  467. # Observations on the SPARQL grammar (http://www.w3.org/TR/2013/REC-sparql11-query-20130321/):
  468. # 1. Only the terminals STRING_LITERAL1, STRING_LITERAL2,
  469. # STRING_LITERAL_LONG1, STRING_LITERAL_LONG2, and comments can contain
  470. # curly braces.
  471. # 2. The non-terminals introduce curly braces in pairs only.
  472. # 3. Unescaped " can occur only in strings and comments.
  473. # 3. Unescaped ' can occur only in strings, comments, and IRIRefs.
  474. # 4. \ always escapes the following character, especially \", \', and
  475. # \\ denote literal ", ', and \ respectively.
  476. # 5. # always starts a comment outside of string and IRI
  477. # 6. A comment ends at the next newline
  478. # 7. IRIREFs need to be detected, as they may contain # without starting a comment
  479. # 8. PrefixedNames do not contain a #
  480. # As a consequence, it should be rather easy to detect strings and comments
  481. # in order to avoid unbalanced curly braces.
  482. # From the SPARQL grammar
  483. STRING_LITERAL1 = "'([^'\\\\]|\\\\.)*'"
  484. STRING_LITERAL2 = '"([^"\\\\]|\\\\.)*"'
  485. STRING_LITERAL_LONG1 = "'''(('|'')?([^'\\\\]|\\\\.))*'''"
  486. STRING_LITERAL_LONG2 = '"""(("|"")?([^"\\\\]|\\\\.))*"""'
  487. String = "(%s)|(%s)|(%s)|(%s)" % (
  488. STRING_LITERAL1,
  489. STRING_LITERAL2,
  490. STRING_LITERAL_LONG1,
  491. STRING_LITERAL_LONG2,
  492. )
  493. IRIREF = '<([^<>"{}|^`\\]\\\\[\\x00-\\x20])*>'
  494. COMMENT = "#[^\\x0D\\x0A]*([\\x0D\\x0A]|\\Z)"
  495. # Simplified grammar to find { at beginning and } at end of blocks
  496. BLOCK_START = "{"
  497. BLOCK_END = "}"
  498. ESCAPED = "\\\\."
  499. # Match anything that doesn't start or end a block:
  500. BlockContent = "(%s)|(%s)|(%s)|(%s)" % (String, IRIREF, COMMENT, ESCAPED)
  501. BlockFinding = "(?P<block_start>%s)|(?P<block_end>%s)|(?P<block_content>%s)" % (
  502. BLOCK_START,
  503. BLOCK_END,
  504. BlockContent,
  505. )
  506. BLOCK_FINDING_PATTERN = re.compile(BlockFinding)
  507. # Note that BLOCK_FINDING_PATTERN.finditer() will not cover the whole
  508. # string with matches. Everything that is not matched will have to be
  509. # part of the modified query as is.
  510. ##################################################################
  511. def __init__(
  512. self,
  513. query_endpoint: Optional[str] = None,
  514. update_endpoint: Optional[str] = None,
  515. sparql11: bool = True,
  516. context_aware: bool = True,
  517. postAsEncoded: bool = True, # noqa: N803
  518. autocommit: bool = True,
  519. dirty_reads: bool = False,
  520. **kwds,
  521. ):
  522. """
  523. Args:
  524. autocommit: if set, the store will commit after every
  525. writing operations. If False, we only make queries on the
  526. server once commit is called.
  527. dirty_reads if set, we do not commit before reading. So you
  528. cannot read what you wrote before manually calling commit.
  529. """
  530. SPARQLStore.__init__(
  531. self,
  532. query_endpoint,
  533. sparql11,
  534. context_aware,
  535. update_endpoint=update_endpoint,
  536. **kwds,
  537. )
  538. self.postAsEncoded = postAsEncoded
  539. self.autocommit = autocommit
  540. self.dirty_reads = dirty_reads
  541. self._edits: Optional[List[str]] = None
  542. self._updates = 0
  543. def query(self, *args: Any, **kwargs: Any) -> Result:
  544. if not self.autocommit and not self.dirty_reads:
  545. self.commit()
  546. return SPARQLStore.query(self, *args, **kwargs)
  547. # type error: Signature of "triples" incompatible with supertype "Store"
  548. def triples( # type: ignore[override]
  549. self, *args: Any, **kwargs: Any
  550. ) -> Iterator[Tuple[_TripleType, None]]:
  551. if not self.autocommit and not self.dirty_reads:
  552. self.commit()
  553. return SPARQLStore.triples(self, *args, **kwargs)
  554. # type error: Signature of "contexts" incompatible with supertype "Store"
  555. def contexts( # type: ignore[override]
  556. self, *args: Any, **kwargs: Any
  557. ) -> Generator[_ContextIdentifierType, None, None]:
  558. if not self.autocommit and not self.dirty_reads:
  559. self.commit()
  560. return SPARQLStore.contexts(self, *args, **kwargs)
  561. def __len__(self, *args: Any, **kwargs: Any) -> int:
  562. if not self.autocommit and not self.dirty_reads:
  563. self.commit()
  564. return SPARQLStore.__len__(self, *args, **kwargs)
  565. def open(
  566. self, configuration: Union[str, Tuple[str, str]], create: bool = False
  567. ) -> None:
  568. """Sets the endpoint URLs for this `SPARQLStore`
  569. Args:
  570. configuration: either a tuple of (query_endpoint, update_endpoint),
  571. or a string with the endpoint which is configured as query and update endpoint
  572. create: if True an exception is thrown.
  573. """
  574. if create:
  575. raise Exception("Cannot create a SPARQL Endpoint")
  576. if isinstance(configuration, tuple):
  577. self.query_endpoint = configuration[0]
  578. if len(configuration) > 1:
  579. self.update_endpoint = configuration[1]
  580. else:
  581. self.query_endpoint = configuration
  582. self.update_endpoint = configuration
  583. def _transaction(self) -> List[str]:
  584. if self._edits is None:
  585. self._edits = []
  586. return self._edits
  587. # Transactional interfaces
  588. def commit(self) -> None:
  589. """`add()`, `addN()`, and `remove()` are transactional to reduce overhead of many small edits.
  590. Read and update() calls will automatically commit any outstanding edits.
  591. This should behave as expected most of the time, except that alternating writes
  592. and reads can degenerate to the original call-per-triple situation that originally existed.
  593. """
  594. if self._edits and len(self._edits) > 0:
  595. self._update("\n;\n".join(self._edits))
  596. self._edits = None
  597. def rollback(self) -> None:
  598. self._edits = None
  599. def add(
  600. self,
  601. spo: _TripleType,
  602. context: Optional[_ContextType] = None,
  603. quoted: bool = False,
  604. ) -> None:
  605. """Add a triple to the store of triples."""
  606. if not self.update_endpoint:
  607. raise Exception("UpdateEndpoint is not set")
  608. assert not quoted
  609. (subject, predicate, obj) = spo
  610. nts = self.node_to_sparql
  611. triple = "%s %s %s ." % (nts(subject), nts(predicate), nts(obj))
  612. if self._is_contextual(context):
  613. if TYPE_CHECKING:
  614. # _is_contextual will never return true if context is None
  615. assert context is not None
  616. q = "INSERT DATA { GRAPH %s { %s } }" % (nts(context.identifier), triple)
  617. else:
  618. q = "INSERT DATA { %s }" % triple
  619. self._transaction().append(q)
  620. if self.autocommit:
  621. self.commit()
  622. def addN(self, quads: Iterable[_QuadType]) -> None: # noqa: N802
  623. """Add a list of quads to the store."""
  624. if not self.update_endpoint:
  625. raise Exception("UpdateEndpoint is not set - call 'open'")
  626. contexts = collections.defaultdict(list)
  627. for subject, predicate, obj, context in quads:
  628. contexts[context].append((subject, predicate, obj))
  629. data: List[str] = []
  630. nts = self.node_to_sparql
  631. for context in contexts:
  632. triples = [
  633. "%s %s %s ." % (nts(subject), nts(predicate), nts(obj))
  634. for subject, predicate, obj in contexts[context]
  635. ]
  636. data.append(
  637. "INSERT DATA { GRAPH %s { %s } }\n"
  638. % (nts(context.identifier), "\n".join(triples))
  639. )
  640. self._transaction().extend(data)
  641. if self.autocommit:
  642. self.commit()
  643. # type error: Signature of "remove" incompatible with supertype "Store"
  644. def remove( # type: ignore[override]
  645. self, spo: _TriplePatternType, context: Optional[_ContextType]
  646. ) -> None:
  647. """Remove a triple from the store"""
  648. if not self.update_endpoint:
  649. raise Exception("UpdateEndpoint is not set - call 'open'")
  650. (subject, predicate, obj) = spo
  651. if not subject:
  652. subject = Variable("S")
  653. if not predicate:
  654. predicate = Variable("P")
  655. if not obj:
  656. obj = Variable("O")
  657. nts = self.node_to_sparql
  658. triple = "%s %s %s ." % (nts(subject), nts(predicate), nts(obj))
  659. if self._is_contextual(context):
  660. if TYPE_CHECKING:
  661. # _is_contextual will never return true if context is None
  662. assert context is not None
  663. cid = nts(context.identifier)
  664. q = "WITH %(graph)s DELETE { %(triple)s } WHERE { %(triple)s }" % {
  665. "graph": cid,
  666. "triple": triple,
  667. }
  668. else:
  669. q = "DELETE { %s } WHERE { %s } " % (triple, triple)
  670. self._transaction().append(q)
  671. if self.autocommit:
  672. self.commit()
  673. def setTimeout(self, timeout) -> None: # noqa: N802
  674. self._timeout = int(timeout)
  675. def _update(self, update):
  676. self._updates += 1
  677. SPARQLConnector.update(self, update)
  678. # type error: Signature of "update" incompatible with supertype "SPARQLConnector"
  679. # type error: Signature of "update" incompatible with supertype "Store"
  680. def update( # type: ignore[override]
  681. self,
  682. query: Union[Update, str],
  683. initNs: Dict[str, Any] = {}, # noqa: N803
  684. initBindings: Dict[str, Identifier] = {}, # noqa: N803
  685. queryGraph: Optional[str] = None, # noqa: N803
  686. DEBUG: bool = False, # noqa: N803
  687. ):
  688. """Perform a SPARQL Update Query against the endpoint, INSERT, LOAD, DELETE etc.
  689. Setting initNs adds PREFIX declarations to the beginning of
  690. the update. Setting initBindings adds inline VALUEs to the
  691. beginning of every WHERE clause. By the SPARQL grammar, all
  692. operations that support variables (namely INSERT and DELETE)
  693. require a WHERE clause.
  694. Important: initBindings fails if the update contains the
  695. substring 'WHERE {' which does not denote a WHERE clause, e.g.
  696. if it is part of a literal.
  697. !!! info "Context-aware query rewriting"
  698. - **When:** If context-awareness is enabled and the graph is not the default graph of the store.
  699. - **Why:** To ensure consistency with the [`Memory`][rdflib.plugins.stores.memory.Memory] store.
  700. The graph must accept "local" SPARQL requests (requests with no GRAPH keyword)
  701. as if it was the default graph.
  702. - **What is done:** These "local" queries are rewritten by this store.
  703. The content of each block of a SPARQL Update operation is wrapped in a GRAPH block
  704. except if the block is empty.
  705. This basically causes INSERT, INSERT DATA, DELETE, DELETE DATA and WHERE to operate
  706. only on the context.
  707. - **Example:** `"INSERT DATA { <urn:michel> <urn:likes> <urn:pizza> }"` is converted into
  708. `"INSERT DATA { GRAPH <urn:graph> { <urn:michel> <urn:likes> <urn:pizza> } }"`.
  709. - **Warning:** Queries are presumed to be "local" but this assumption is **not checked**.
  710. For instance, if the query already contains GRAPH blocks, the latter will be wrapped in new GRAPH blocks.
  711. - **Warning:** A simplified grammar is used that should tolerate
  712. extensions of the SPARQL grammar. Still, the process may fail in
  713. uncommon situations and produce invalid output.
  714. """
  715. if not self.update_endpoint:
  716. raise Exception("Update endpoint is not set!")
  717. self.debug = DEBUG
  718. assert isinstance(query, str)
  719. query = self._inject_prefixes(query, initNs)
  720. if self._is_contextual(queryGraph):
  721. if TYPE_CHECKING:
  722. # _is_contextual will never return true if context is None
  723. assert queryGraph is not None
  724. query = self._insert_named_graph(query, queryGraph)
  725. if initBindings:
  726. # For INSERT and DELETE the WHERE clause is obligatory
  727. # (http://www.w3.org/TR/2013/REC-sparql11-query-20130321/#rModify)
  728. # Other query types do not allow variables and don't
  729. # have a WHERE clause. This also works for updates with
  730. # more than one INSERT/DELETE.
  731. v = list(initBindings)
  732. values = "\nVALUES ( %s )\n{ ( %s ) }\n" % (
  733. " ".join("?" + str(x) for x in v),
  734. " ".join(self.node_to_sparql(initBindings[x]) for x in v),
  735. )
  736. query = self.where_pattern.sub("WHERE { " + values, query)
  737. self._transaction().append(query)
  738. if self.autocommit:
  739. self.commit()
  740. def _insert_named_graph(self, query: str, query_graph: str) -> str:
  741. """Inserts GRAPH <query_graph> {} into blocks of SPARQL Update operations
  742. For instance, `INSERT DATA { <urn:michel> <urn:likes> <urn:pizza> }`
  743. is converted into
  744. `INSERT DATA { GRAPH <urn:graph> { <urn:michel> <urn:likes> <urn:pizza> } }`
  745. """
  746. if isinstance(query_graph, Node):
  747. query_graph = self.node_to_sparql(query_graph)
  748. else:
  749. query_graph = "<%s>" % query_graph
  750. graph_block_open = " GRAPH %s {" % query_graph
  751. graph_block_close = "} "
  752. # SPARQL Update supports the following operations:
  753. # LOAD, CLEAR, DROP, ADD, MOVE, COPY, CREATE, INSERT DATA, DELETE DATA, DELETE/INSERT, DELETE WHERE
  754. # LOAD, CLEAR, DROP, ADD, MOVE, COPY, CREATE do not make much sense in a context.
  755. # INSERT DATA, DELETE DATA, and DELETE WHERE require the contents of their block to be wrapped in a GRAPH <?> { }.
  756. # DELETE/INSERT supports the WITH keyword, which sets the graph to be
  757. # used for all following DELETE/INSERT instruction including the
  758. # non-optional WHERE block. Equivalently, a GRAPH block can be added to
  759. # all blocks.
  760. #
  761. # Strategy employed here: Wrap the contents of every top-level block into a `GRAPH <?> { }`.
  762. level = 0
  763. modified_query = []
  764. pos = 0
  765. for match in self.BLOCK_FINDING_PATTERN.finditer(query):
  766. if match.group("block_start") is not None:
  767. level += 1
  768. if level == 1:
  769. modified_query.append(query[pos : match.end()])
  770. modified_query.append(graph_block_open)
  771. pos = match.end()
  772. elif match.group("block_end") is not None:
  773. if level == 1:
  774. since_previous_pos = query[pos : match.start()]
  775. if modified_query[-1] is graph_block_open and (
  776. since_previous_pos == "" or since_previous_pos.isspace()
  777. ):
  778. # In this case, adding graph_block_start and
  779. # graph_block_end results in an empty GRAPH block. Some
  780. # endpoints (e.g. TDB) can not handle this. Therefore
  781. # remove the previously added block_start.
  782. modified_query.pop()
  783. modified_query.append(since_previous_pos)
  784. else:
  785. modified_query.append(since_previous_pos)
  786. modified_query.append(graph_block_close)
  787. pos = match.start()
  788. level -= 1
  789. modified_query.append(query[pos:])
  790. return "".join(modified_query)
  791. def add_graph(self, graph: Graph) -> None:
  792. if not self.graph_aware:
  793. Store.add_graph(self, graph)
  794. elif graph.identifier != DATASET_DEFAULT_GRAPH_ID:
  795. self.update("CREATE GRAPH %s" % self.node_to_sparql(graph.identifier))
  796. def remove_graph(self, graph: Graph) -> None:
  797. if not self.graph_aware:
  798. Store.remove_graph(self, graph)
  799. elif graph.identifier == DATASET_DEFAULT_GRAPH_ID:
  800. self.update("DROP DEFAULT")
  801. else:
  802. self.update("DROP GRAPH %s" % self.node_to_sparql(graph.identifier))
  803. def subjects(
  804. self,
  805. predicate: Optional[_PredicateType] = None,
  806. object: Optional[_ObjectType] = None,
  807. ) -> Generator[_SubjectType, None, None]:
  808. """A generator of subjects with the given predicate and object"""
  809. for t, c in self.triples((None, predicate, object)):
  810. yield t[0]
  811. def predicates(
  812. self,
  813. subject: Optional[_SubjectType] = None,
  814. object: Optional[_ObjectType] = None,
  815. ) -> Generator[_PredicateType, None, None]:
  816. """A generator of predicates with the given subject and object"""
  817. for t, c in self.triples((subject, None, object)):
  818. yield t[1]
  819. def objects(
  820. self,
  821. subject: Optional[_SubjectType] = None,
  822. predicate: Optional[_PredicateType] = None,
  823. ) -> Generator[_ObjectType, None, None]:
  824. """A generator of objects with the given subject and predicate"""
  825. for t, c in self.triples((subject, predicate, None)):
  826. yield t[2]
  827. def subject_predicates(
  828. self, object: Optional[_ObjectType] = None
  829. ) -> Generator[Tuple[_SubjectType, _PredicateType], None, None]:
  830. """A generator of (subject, predicate) tuples for the given object"""
  831. for t, c in self.triples((None, None, object)):
  832. yield t[0], t[1]
  833. def subject_objects(
  834. self, predicate: Optional[_PredicateType] = None
  835. ) -> Generator[Tuple[_SubjectType, _ObjectType], None, None]:
  836. """A generator of (subject, object) tuples for the given predicate"""
  837. for t, c in self.triples((None, predicate, None)):
  838. yield t[0], t[2]
  839. def predicate_objects(
  840. self, subject: Optional[_SubjectType] = None
  841. ) -> Generator[Tuple[_PredicateType, _ObjectType], None, None]:
  842. """A generator of (predicate, object) tuples for the given subject"""
  843. for t, c in self.triples((subject, None, None)):
  844. yield t[1], t[2]
  845. __all__ = ["SPARQLUpdateStore", "SPARQLStore"]