algebra.py 60 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688
  1. """
  2. Converting the 'parse-tree' output of pyparsing to a SPARQL Algebra expression
  3. http://www.w3.org/TR/sparql11-query/#sparqlQuery
  4. """
  5. from __future__ import annotations
  6. import collections
  7. import functools
  8. import operator
  9. import typing
  10. from functools import reduce
  11. from typing import (
  12. Any,
  13. Callable,
  14. DefaultDict,
  15. Dict,
  16. Iterable,
  17. List,
  18. Mapping,
  19. Optional,
  20. Set,
  21. Tuple,
  22. overload,
  23. )
  24. from pyparsing import ParseResults
  25. from rdflib.paths import (
  26. AlternativePath,
  27. InvPath,
  28. MulPath,
  29. NegatedPath,
  30. Path,
  31. SequencePath,
  32. )
  33. from rdflib.plugins.sparql.operators import TrueFilter, and_
  34. from rdflib.plugins.sparql.operators import simplify as simplifyFilters
  35. from rdflib.plugins.sparql.parserutils import CompValue, Expr
  36. from rdflib.plugins.sparql.sparql import Prologue, Query, Update
  37. # ---------------------------
  38. # Some convenience methods
  39. from rdflib.term import BNode, Identifier, Literal, URIRef, Variable
  40. def OrderBy(p: CompValue, expr: List[CompValue]) -> CompValue:
  41. return CompValue("OrderBy", p=p, expr=expr)
  42. def ToMultiSet(p: typing.Union[List[Dict[Variable, str]], CompValue]) -> CompValue:
  43. return CompValue("ToMultiSet", p=p)
  44. def Union(p1: CompValue, p2: CompValue) -> CompValue:
  45. return CompValue("Union", p1=p1, p2=p2)
  46. def Join(p1: CompValue, p2: Optional[CompValue]) -> CompValue:
  47. return CompValue("Join", p1=p1, p2=p2)
  48. def Minus(p1: CompValue, p2: CompValue) -> CompValue:
  49. return CompValue("Minus", p1=p1, p2=p2)
  50. def Graph(term: Identifier, graph: CompValue) -> CompValue:
  51. return CompValue("Graph", term=term, p=graph)
  52. def BGP(
  53. triples: Optional[List[Tuple[Identifier, Identifier, Identifier]]] = None
  54. ) -> CompValue:
  55. return CompValue("BGP", triples=triples or [])
  56. def LeftJoin(p1: CompValue, p2: CompValue, expr) -> CompValue:
  57. return CompValue("LeftJoin", p1=p1, p2=p2, expr=expr)
  58. def Filter(expr: Expr, p: CompValue) -> CompValue:
  59. return CompValue("Filter", expr=expr, p=p)
  60. def Extend(
  61. p: CompValue, expr: typing.Union[Identifier, Expr], var: Variable
  62. ) -> CompValue:
  63. return CompValue("Extend", p=p, expr=expr, var=var)
  64. def Values(res: List[Dict[Variable, str]]) -> CompValue:
  65. return CompValue("values", res=res)
  66. def Project(p: CompValue, PV: List[Variable]) -> CompValue:
  67. return CompValue("Project", p=p, PV=PV)
  68. def Group(p: CompValue, expr: Optional[List[Variable]] = None) -> CompValue:
  69. return CompValue("Group", p=p, expr=expr)
  70. def _knownTerms(
  71. triple: Tuple[Identifier, Identifier, Identifier],
  72. varsknown: Set[typing.Union[BNode, Variable]],
  73. varscount: Dict[Identifier, int],
  74. ) -> Tuple[int, int, bool]:
  75. return (
  76. len(
  77. [
  78. x
  79. for x in triple
  80. if x not in varsknown and isinstance(x, (Variable, BNode))
  81. ]
  82. ),
  83. -sum(varscount.get(x, 0) for x in triple),
  84. not isinstance(triple[2], Literal),
  85. )
  86. def reorderTriples(
  87. l_: Iterable[Tuple[Identifier, Identifier, Identifier]]
  88. ) -> List[Tuple[Identifier, Identifier, Identifier]]:
  89. """
  90. Reorder triple patterns so that we execute the
  91. ones with most bindings first
  92. """
  93. def _addvar(term: str, varsknown: Set[typing.Union[Variable, BNode]]):
  94. if isinstance(term, (Variable, BNode)):
  95. varsknown.add(term)
  96. # NOTE on type errors: most of these are because the same variable is used
  97. # for different types.
  98. # type error: List comprehension has incompatible type List[Tuple[None, Tuple[Identifier, Identifier, Identifier]]]; expected List[Tuple[Identifier, Identifier, Identifier]]
  99. l_ = [(None, x) for x in l_] # type: ignore[misc]
  100. varsknown: Set[typing.Union[BNode, Variable]] = set()
  101. varscount: Dict[Identifier, int] = collections.defaultdict(int)
  102. for t in l_:
  103. for c in t[1]:
  104. if isinstance(c, (Variable, BNode)):
  105. varscount[c] += 1
  106. i = 0
  107. # Done in steps, sort by number of bound terms
  108. # the top block of patterns with the most bound terms is kept
  109. # the rest is resorted based on the vars bound after the first
  110. # block is evaluated
  111. # we sort by decorate/undecorate, since we need the value of the sort keys
  112. while i < len(l_):
  113. # type error: Generator has incompatible item type "Tuple[Any, Identifier]"; expected "Tuple[Identifier, Identifier, Identifier]"
  114. # type error: Argument 1 to "_knownTerms" has incompatible type "Identifier"; expected "Tuple[Identifier, Identifier, Identifier]"
  115. l_[i:] = sorted((_knownTerms(x[1], varsknown, varscount), x[1]) for x in l_[i:]) # type: ignore[misc,arg-type]
  116. # type error: Incompatible types in assignment (expression has type "str", variable has type "Tuple[Identifier, Identifier, Identifier]")
  117. t = l_[i][0][0] # type: ignore[assignment] # top block has this many terms bound
  118. j = 0
  119. while i + j < len(l_) and l_[i + j][0][0] == t:
  120. for c in l_[i + j][1]:
  121. _addvar(c, varsknown)
  122. j += 1
  123. i += 1
  124. # type error: List comprehension has incompatible type List[Identifier]; expected List[Tuple[Identifier, Identifier, Identifier]]
  125. return [x[1] for x in l_] # type: ignore[misc]
  126. def triples(
  127. l: typing.Union[ # noqa: E741
  128. List[List[Identifier]], List[Tuple[Identifier, Identifier, Identifier]]
  129. ]
  130. ) -> List[Tuple[Identifier, Identifier, Identifier]]:
  131. _l = reduce(lambda x, y: x + y, l)
  132. if (len(_l) % 3) != 0:
  133. raise Exception("these aint triples")
  134. return reorderTriples((_l[x], _l[x + 1], _l[x + 2]) for x in range(0, len(_l), 3))
  135. # type error: Missing return statement
  136. def translatePName( # type: ignore[return]
  137. p: typing.Union[CompValue, str], prologue: Prologue
  138. ) -> Optional[Identifier]:
  139. """
  140. Expand prefixed/relative URIs
  141. """
  142. if isinstance(p, CompValue):
  143. if p.name == "pname":
  144. # type error: Incompatible return value type (got "Union[CompValue, str, None]", expected "Optional[Identifier]")
  145. return prologue.absolutize(p) # type: ignore[return-value]
  146. if p.name == "literal":
  147. # type error: Argument "datatype" to "Literal" has incompatible type "Union[CompValue, str, None]"; expected "Optional[str]"
  148. return Literal(
  149. p.string, lang=p.lang, datatype=prologue.absolutize(p.datatype) # type: ignore[arg-type]
  150. )
  151. elif isinstance(p, URIRef):
  152. # type error: Incompatible return value type (got "Union[CompValue, str, None]", expected "Optional[Identifier]")
  153. return prologue.absolutize(p) # type: ignore[return-value]
  154. @overload
  155. def translatePath(p: URIRef) -> None: ...
  156. @overload
  157. def translatePath(p: CompValue) -> Path: ...
  158. # type error: Missing return statement
  159. def translatePath(p: typing.Union[CompValue, URIRef]) -> Optional[Path]: # type: ignore[return]
  160. """
  161. Translate PropertyPath expressions
  162. """
  163. if isinstance(p, CompValue):
  164. if p.name == "PathAlternative":
  165. if len(p.part) == 1:
  166. return p.part[0]
  167. else:
  168. return AlternativePath(*p.part)
  169. elif p.name == "PathSequence":
  170. if len(p.part) == 1:
  171. return p.part[0]
  172. else:
  173. return SequencePath(*p.part)
  174. elif p.name == "PathElt":
  175. if not p.mod:
  176. return p.part
  177. else:
  178. if isinstance(p.part, list):
  179. if len(p.part) != 1:
  180. raise Exception("Denkfehler!")
  181. return MulPath(p.part[0], p.mod)
  182. else:
  183. return MulPath(p.part, p.mod)
  184. elif p.name == "PathEltOrInverse":
  185. if isinstance(p.part, list):
  186. if len(p.part) != 1:
  187. raise Exception("Denkfehler!")
  188. return InvPath(p.part[0])
  189. else:
  190. return InvPath(p.part)
  191. elif p.name == "PathNegatedPropertySet":
  192. if isinstance(p.part, list):
  193. return NegatedPath(AlternativePath(*p.part))
  194. else:
  195. return NegatedPath(p.part)
  196. def translateExists(
  197. e: typing.Union[Expr, Literal, Variable, URIRef]
  198. ) -> typing.Union[Expr, Literal, Variable, URIRef]:
  199. """
  200. Translate the graph pattern used by EXISTS and NOT EXISTS
  201. http://www.w3.org/TR/sparql11-query/#sparqlCollectFilters
  202. """
  203. def _c(n):
  204. if isinstance(n, CompValue):
  205. if n.name in ("Builtin_EXISTS", "Builtin_NOTEXISTS"):
  206. n.graph = translateGroupGraphPattern(n.graph)
  207. if n.graph.name == "Filter":
  208. # filters inside (NOT) EXISTS can see vars bound outside
  209. n.graph.no_isolated_scope = True
  210. e = traverse(e, visitPost=_c)
  211. return e
  212. def collectAndRemoveFilters(parts: List[CompValue]) -> Optional[Expr]:
  213. """FILTER expressions apply to the whole group graph pattern in which
  214. they appear.
  215. http://www.w3.org/TR/sparql11-query/#sparqlCollectFilters
  216. """
  217. filters = []
  218. i = 0
  219. while i < len(parts):
  220. p = parts[i]
  221. if p.name == "Filter":
  222. filters.append(translateExists(p.expr))
  223. parts.pop(i)
  224. else:
  225. i += 1
  226. if filters:
  227. # type error: Argument 1 to "and_" has incompatible type "*List[Union[Expr, Literal, Variable]]"; expected "Expr"
  228. return and_(*filters) # type: ignore[arg-type]
  229. return None
  230. def translateGroupOrUnionGraphPattern(graphPattern: CompValue) -> Optional[CompValue]:
  231. A: Optional[CompValue] = None
  232. for g in graphPattern.graph:
  233. g = translateGroupGraphPattern(g)
  234. if not A:
  235. A = g
  236. else:
  237. A = Union(A, g)
  238. return A
  239. def translateGraphGraphPattern(graphPattern: CompValue) -> CompValue:
  240. return Graph(graphPattern.term, translateGroupGraphPattern(graphPattern.graph))
  241. def translateInlineData(graphPattern: CompValue) -> CompValue:
  242. return ToMultiSet(translateValues(graphPattern))
  243. def translateGroupGraphPattern(graphPattern: CompValue) -> CompValue:
  244. """
  245. http://www.w3.org/TR/sparql11-query/#convertGraphPattern
  246. """
  247. if graphPattern.translated:
  248. # This occurs if it is attempted to translate a group graph pattern twice,
  249. # which occurs with nested (NOT) EXISTS filters. Simply return the already
  250. # translated pattern instead.
  251. return graphPattern
  252. if graphPattern.name == "SubSelect":
  253. # The first output from translate cannot be None for a subselect query
  254. # as it can only be None for certain DESCRIBE queries.
  255. # type error: Argument 1 to "ToMultiSet" has incompatible type "Optional[CompValue]";
  256. # expected "Union[List[Dict[Variable, str]], CompValue]"
  257. return ToMultiSet(translate(graphPattern)[0]) # type: ignore[arg-type]
  258. if not graphPattern.part:
  259. graphPattern.part = [] # empty { }
  260. filters = collectAndRemoveFilters(graphPattern.part)
  261. g: List[CompValue] = []
  262. for p in graphPattern.part:
  263. if p.name == "TriplesBlock":
  264. # merge adjacent TripleBlocks
  265. if not (g and g[-1].name == "BGP"):
  266. g.append(BGP())
  267. g[-1]["triples"] += triples(p.triples)
  268. else:
  269. g.append(p)
  270. G = BGP()
  271. for p in g:
  272. if p.name == "OptionalGraphPattern":
  273. A = translateGroupGraphPattern(p.graph)
  274. if A.name == "Filter":
  275. G = LeftJoin(G, A.p, A.expr)
  276. else:
  277. G = LeftJoin(G, A, TrueFilter)
  278. elif p.name == "MinusGraphPattern":
  279. G = Minus(p1=G, p2=translateGroupGraphPattern(p.graph))
  280. elif p.name == "GroupOrUnionGraphPattern":
  281. G = Join(p1=G, p2=translateGroupOrUnionGraphPattern(p))
  282. elif p.name == "GraphGraphPattern":
  283. G = Join(p1=G, p2=translateGraphGraphPattern(p))
  284. elif p.name == "InlineData":
  285. G = Join(p1=G, p2=translateInlineData(p))
  286. elif p.name == "ServiceGraphPattern":
  287. G = Join(p1=G, p2=p)
  288. elif p.name in ("BGP", "Extend"):
  289. G = Join(p1=G, p2=p)
  290. elif p.name == "Bind":
  291. # translateExists will translate the expression if it is EXISTS, and otherwise return
  292. # the expression as is. This is needed because EXISTS has a graph pattern
  293. # which must be translated to work properly during evaluation.
  294. G = Extend(G, translateExists(p.expr), p.var)
  295. else:
  296. raise Exception(
  297. "Unknown part in GroupGraphPattern: %s - %s" % (type(p), p.name)
  298. )
  299. if filters:
  300. G = Filter(expr=filters, p=G)
  301. # Mark this graph pattern as translated
  302. G.translated = True
  303. return G
  304. class StopTraversal(Exception): # noqa: N818
  305. def __init__(self, rv: bool):
  306. self.rv = rv
  307. def _traverse(
  308. e: Any,
  309. visitPre: Callable[[Any], Any] = lambda n: None,
  310. visitPost: Callable[[Any], Any] = lambda n: None,
  311. ):
  312. """Traverse a parse-tree, visit each node
  313. if visit functions return a value, replace current node
  314. """
  315. _e = visitPre(e)
  316. if _e is not None:
  317. return _e
  318. if e is None:
  319. return None
  320. if isinstance(e, (list, ParseResults)):
  321. return [_traverse(x, visitPre, visitPost) for x in e]
  322. elif isinstance(e, tuple):
  323. return tuple([_traverse(x, visitPre, visitPost) for x in e])
  324. elif isinstance(e, CompValue):
  325. for k, val in e.items():
  326. e[k] = _traverse(val, visitPre, visitPost)
  327. _e = visitPost(e)
  328. if _e is not None:
  329. return _e
  330. return e
  331. def _traverseAgg(e: Any, visitor: Callable[[Any, Any], Any] = lambda n, v: None):
  332. """
  333. Traverse a parse-tree, visit each node
  334. if visit functions return a value, replace current node
  335. """
  336. res = []
  337. if isinstance(e, (list, ParseResults, tuple)):
  338. res = [_traverseAgg(x, visitor) for x in e]
  339. elif isinstance(e, CompValue):
  340. for k, val in e.items():
  341. if val is not None:
  342. res.append(_traverseAgg(val, visitor))
  343. return visitor(e, res)
  344. def traverse(
  345. tree,
  346. visitPre: Callable[[Any], Any] = lambda n: None,
  347. visitPost: Callable[[Any], Any] = lambda n: None,
  348. complete: Optional[bool] = None,
  349. ) -> Any:
  350. """
  351. Traverse tree, visit each node with visit function
  352. visit function may raise StopTraversal to stop traversal
  353. if complete!=None, it is returned on complete traversal,
  354. otherwise the transformed tree is returned
  355. """
  356. try:
  357. r = _traverse(tree, visitPre, visitPost)
  358. if complete is not None:
  359. return complete
  360. return r
  361. except StopTraversal as st:
  362. return st.rv
  363. def _hasAggregate(x) -> None:
  364. """
  365. Traverse parse(sub)Tree
  366. return true if any aggregates are used
  367. """
  368. if isinstance(x, CompValue):
  369. if x.name.startswith("Aggregate_"):
  370. raise StopTraversal(True)
  371. # type error: Missing return statement
  372. def _aggs(e, A) -> Optional[Variable]: # type: ignore[return]
  373. """
  374. Collect Aggregates in A
  375. replaces aggregates with variable references
  376. """
  377. # TODO: nested Aggregates?
  378. if isinstance(e, CompValue) and e.name.startswith("Aggregate_"):
  379. A.append(e)
  380. aggvar = Variable("__agg_%d__" % len(A))
  381. e["res"] = aggvar
  382. return aggvar
  383. # type error: Missing return statement
  384. def _findVars(x, res: Set[Variable]) -> Optional[CompValue]: # type: ignore[return]
  385. """
  386. Find all variables in a tree
  387. """
  388. if isinstance(x, Variable):
  389. res.add(x)
  390. if isinstance(x, CompValue):
  391. if x.name == "Bind":
  392. res.add(x.var)
  393. return x # stop recursion and finding vars in the expr
  394. elif x.name == "SubSelect":
  395. if x.projection:
  396. res.update(v.var or v.evar for v in x.projection)
  397. return x
  398. def _addVars(x, children: List[Set[Variable]]) -> Set[Variable]:
  399. """
  400. find which variables may be bound by this part of the query
  401. """
  402. if isinstance(x, Variable):
  403. return set([x])
  404. elif isinstance(x, CompValue):
  405. if x.name == "RelationalExpression":
  406. x["_vars"] = set()
  407. elif x.name == "Extend":
  408. # vars only used in the expr for a bind should not be included
  409. x["_vars"] = reduce(
  410. operator.or_,
  411. [child for child, part in zip(children, x) if part != "expr"],
  412. set(),
  413. )
  414. else:
  415. x["_vars"] = set(reduce(operator.or_, children, set()))
  416. if x.name == "SubSelect":
  417. if x.projection:
  418. s = set(v.var or v.evar for v in x.projection)
  419. else:
  420. s = set()
  421. return s
  422. return x["_vars"]
  423. return reduce(operator.or_, children, set())
  424. # type error: Missing return statement
  425. def _sample(e: typing.Union[CompValue, List[Expr], Expr, List[str], Variable], v: Optional[Variable] = None) -> Optional[CompValue]: # type: ignore[return]
  426. """
  427. For each unaggregated variable V in expr
  428. Replace V with Sample(V)
  429. """
  430. if isinstance(e, CompValue) and e.name.startswith("Aggregate_"):
  431. return e # do not replace vars in aggregates
  432. if isinstance(e, Variable) and v != e:
  433. return CompValue("Aggregate_Sample", vars=e)
  434. def _simplifyFilters(e: Any) -> Any:
  435. if isinstance(e, Expr):
  436. return simplifyFilters(e)
  437. def translateAggregates(
  438. q: CompValue, M: CompValue
  439. ) -> Tuple[CompValue, List[Tuple[Variable, Variable]]]:
  440. E: List[Tuple[Variable, Variable]] = []
  441. A: List[CompValue] = []
  442. # collect/replace aggs in :
  443. # select expr as ?var
  444. if q.projection:
  445. for v in q.projection:
  446. if v.evar:
  447. v.expr = traverse(v.expr, functools.partial(_sample, v=v.evar))
  448. v.expr = traverse(v.expr, functools.partial(_aggs, A=A))
  449. # having clause
  450. if traverse(q.having, _hasAggregate, complete=True):
  451. q.having = traverse(q.having, _sample)
  452. traverse(q.having, functools.partial(_aggs, A=A))
  453. # order by
  454. if traverse(q.orderby, _hasAggregate, complete=False):
  455. q.orderby = traverse(q.orderby, _sample)
  456. traverse(q.orderby, functools.partial(_aggs, A=A))
  457. # sample all other select vars
  458. # TODO: only allowed for vars in group-by?
  459. if q.projection:
  460. for v in q.projection:
  461. if v.var:
  462. rv = Variable("__agg_%d__" % (len(A) + 1))
  463. A.append(CompValue("Aggregate_Sample", vars=v.var, res=rv))
  464. E.append((rv, v.var))
  465. return CompValue("AggregateJoin", A=A, p=M), E
  466. def translateValues(
  467. v: CompValue,
  468. ) -> typing.Union[List[Dict[Variable, str]], CompValue]:
  469. # if len(v.var)!=len(v.value):
  470. # raise Exception("Unmatched vars and values in ValueClause: "+str(v))
  471. res: List[Dict[Variable, str]] = []
  472. if not v.var:
  473. return res
  474. if not v.value:
  475. return res
  476. if not isinstance(v.value[0], list):
  477. for val in v.value:
  478. res.append({v.var[0]: val})
  479. else:
  480. for vals in v.value:
  481. res.append(dict(zip(v.var, vals)))
  482. return Values(res)
  483. def translate(q: CompValue) -> Tuple[Optional[CompValue], List[Variable]]:
  484. """
  485. http://www.w3.org/TR/sparql11-query/#convertSolMod
  486. """
  487. _traverse(q, _simplifyFilters)
  488. q.where = traverse(q.where, visitPost=translatePath)
  489. # TODO: Var scope test
  490. VS: Set[Variable] = set()
  491. # All query types have a WHERE clause EXCEPT some DESCRIBE queries
  492. # where only explicit IRIs are provided.
  493. if q.name == "DescribeQuery":
  494. # For DESCRIBE queries, use the vars provided in q.var.
  495. # If there is no WHERE clause, vars should be explicit IRIs to describe.
  496. # If there is a WHERE clause, vars can be any combination of explicit IRIs
  497. # and variables.
  498. VS = set(q.var)
  499. # If there is no WHERE clause, just return the vars projected
  500. if q.where is None:
  501. return None, list(VS)
  502. # Otherwise, evaluate the WHERE clause like SELECT DISTINCT
  503. else:
  504. q.modifier = "DISTINCT"
  505. else:
  506. traverse(q.where, functools.partial(_findVars, res=VS))
  507. # depth-first recursive generation of mapped query tree
  508. M = translateGroupGraphPattern(q.where)
  509. aggregate = False
  510. if q.groupby:
  511. conditions = []
  512. # convert "GROUP BY (?expr as ?var)" to an Extend
  513. for c in q.groupby.condition:
  514. if isinstance(c, CompValue) and c.name == "GroupAs":
  515. M = Extend(M, c.expr, c.var)
  516. c = c.var
  517. conditions.append(c)
  518. M = Group(p=M, expr=conditions)
  519. aggregate = True
  520. elif (
  521. traverse(q.having, _hasAggregate, complete=False)
  522. or traverse(q.orderby, _hasAggregate, complete=False)
  523. or any(
  524. traverse(x.expr, _hasAggregate, complete=False)
  525. for x in q.projection or []
  526. if x.evar
  527. )
  528. ):
  529. # if any aggregate is used, implicit group by
  530. M = Group(p=M)
  531. aggregate = True
  532. if aggregate:
  533. M, aggregateAliases = translateAggregates(q, M)
  534. else:
  535. aggregateAliases = []
  536. # Need to remove the aggregate var aliases before joining to VALUES;
  537. # else the variable names won't match up correctly when aggregating.
  538. for alias, var in aggregateAliases:
  539. M = Extend(M, alias, var)
  540. # HAVING
  541. if q.having:
  542. M = Filter(expr=and_(*q.having.condition), p=M)
  543. # VALUES
  544. if q.valuesClause:
  545. M = Join(p1=M, p2=ToMultiSet(translateValues(q.valuesClause)))
  546. if not q.projection:
  547. # select *
  548. # Find the first child projection in each branch of the mapped query tree,
  549. # then include the variables it projects out in our projected variables.
  550. for child_projection in _find_first_child_projections(M):
  551. VS |= set(child_projection.PV)
  552. PV = list(VS)
  553. else:
  554. E = list()
  555. PV = list()
  556. for v in q.projection:
  557. if v.var:
  558. if v not in PV:
  559. PV.append(v.var)
  560. elif v.evar:
  561. if v not in PV:
  562. PV.append(v.evar)
  563. E.append((v.expr, v.evar))
  564. else:
  565. raise Exception("I expected a var or evar here!")
  566. for e, v in E:
  567. M = Extend(M, e, v)
  568. # ORDER BY
  569. if q.orderby:
  570. M = OrderBy(
  571. M,
  572. [
  573. CompValue("OrderCondition", expr=c.expr, order=c.order)
  574. for c in q.orderby.condition
  575. ],
  576. )
  577. # PROJECT
  578. M = Project(M, PV)
  579. if q.modifier:
  580. if q.modifier == "DISTINCT":
  581. M = CompValue("Distinct", p=M)
  582. elif q.modifier == "REDUCED":
  583. M = CompValue("Reduced", p=M)
  584. if q.limitoffset:
  585. offset = 0
  586. if q.limitoffset.offset is not None:
  587. offset = q.limitoffset.offset.toPython()
  588. if q.limitoffset.limit is not None:
  589. M = CompValue(
  590. "Slice", p=M, start=offset, length=q.limitoffset.limit.toPython()
  591. )
  592. else:
  593. M = CompValue("Slice", p=M, start=offset)
  594. return M, PV
  595. def _find_first_child_projections(M: CompValue) -> Iterable[CompValue]:
  596. """
  597. Recursively find the first child instance of a Projection operation in each of
  598. the branches of the query execution plan/tree.
  599. """
  600. for child_op in M.values():
  601. if isinstance(child_op, CompValue):
  602. if child_op.name == "Project":
  603. yield child_op
  604. else:
  605. for child_projection in _find_first_child_projections(child_op):
  606. yield child_projection
  607. # type error: Missing return statement
  608. def simplify(n: Any) -> Optional[CompValue]: # type: ignore[return]
  609. """Remove joins to empty BGPs"""
  610. if isinstance(n, CompValue):
  611. if n.name == "Join":
  612. if n.p1.name == "BGP" and len(n.p1.triples) == 0:
  613. return n.p2
  614. if n.p2.name == "BGP" and len(n.p2.triples) == 0:
  615. return n.p1
  616. elif n.name == "BGP":
  617. n["triples"] = reorderTriples(n.triples)
  618. return n
  619. def analyse(n: Any, children: Any) -> bool:
  620. """
  621. Some things can be lazily joined.
  622. This propegates whether they can up the tree
  623. and sets lazy flags for all joins
  624. """
  625. if isinstance(n, CompValue):
  626. if n.name == "Join":
  627. n["lazy"] = all(children)
  628. return False
  629. elif n.name in ("Slice", "Distinct"):
  630. return False
  631. else:
  632. return all(children)
  633. else:
  634. return True
  635. def translatePrologue(
  636. p: ParseResults,
  637. base: Optional[str],
  638. initNs: Optional[Mapping[str, Any]] = None,
  639. prologue: Optional[Prologue] = None,
  640. ) -> Prologue:
  641. if prologue is None:
  642. prologue = Prologue()
  643. prologue.base = ""
  644. if base:
  645. prologue.base = base
  646. if initNs:
  647. for k, v in initNs.items():
  648. prologue.bind(k, v)
  649. x: CompValue
  650. for x in p:
  651. if x.name == "Base":
  652. prologue.base = x.iri
  653. elif x.name == "PrefixDecl":
  654. prologue.bind(x.prefix, prologue.absolutize(x.iri))
  655. return prologue
  656. def translateQuads(
  657. quads: CompValue,
  658. ) -> Tuple[
  659. List[Tuple[Identifier, Identifier, Identifier]],
  660. DefaultDict[str, List[Tuple[Identifier, Identifier, Identifier]]],
  661. ]:
  662. if quads.triples:
  663. alltriples = triples(quads.triples)
  664. else:
  665. alltriples = []
  666. allquads: DefaultDict[str, List[Tuple[Identifier, Identifier, Identifier]]] = (
  667. collections.defaultdict(list)
  668. )
  669. if quads.quadsNotTriples:
  670. for q in quads.quadsNotTriples:
  671. if q.triples:
  672. allquads[q.term] += triples(q.triples)
  673. return alltriples, allquads
  674. def translateUpdate1(u: CompValue, prologue: Prologue) -> CompValue:
  675. if u.name in ("Load", "Clear", "Drop", "Create"):
  676. pass # no translation needed
  677. elif u.name in ("Add", "Move", "Copy"):
  678. pass
  679. elif u.name in ("InsertData", "DeleteData", "DeleteWhere"):
  680. t, q = translateQuads(u.quads)
  681. u["quads"] = q
  682. u["triples"] = t
  683. if u.name in ("DeleteWhere", "DeleteData"):
  684. pass # TODO: check for bnodes in triples
  685. elif u.name == "Modify":
  686. if u.delete:
  687. u.delete["triples"], u.delete["quads"] = translateQuads(u.delete.quads)
  688. if u.insert:
  689. u.insert["triples"], u.insert["quads"] = translateQuads(u.insert.quads)
  690. u["where"] = translateGroupGraphPattern(u.where)
  691. else:
  692. raise Exception("Unknown type of update operation: %s" % u)
  693. u.prologue = prologue
  694. return u
  695. def translateUpdate(
  696. q: CompValue,
  697. base: Optional[str] = None,
  698. initNs: Optional[Mapping[str, Any]] = None,
  699. ) -> Update:
  700. """
  701. Returns a list of SPARQL Update Algebra expressions
  702. """
  703. res: List[CompValue] = []
  704. prologue = None
  705. if not q.request:
  706. # type error: Incompatible return value type (got "List[CompValue]", expected "Update")
  707. return res # type: ignore[return-value]
  708. for p, u in zip(q.prologue, q.request):
  709. prologue = translatePrologue(p, base, initNs, prologue)
  710. # absolutize/resolve prefixes
  711. u = traverse(u, visitPost=functools.partial(translatePName, prologue=prologue))
  712. u = _traverse(u, _simplifyFilters)
  713. u = traverse(u, visitPost=translatePath)
  714. res.append(translateUpdate1(u, prologue))
  715. # type error: Argument 1 to "Update" has incompatible type "Optional[Any]"; expected "Prologue"
  716. return Update(prologue, res) # type: ignore[arg-type]
  717. def translateQuery(
  718. q: ParseResults,
  719. base: Optional[str] = None,
  720. initNs: Optional[Mapping[str, Any]] = None,
  721. ) -> Query:
  722. """
  723. Translate a query-parsetree to a SPARQL Algebra Expression
  724. Return a rdflib.plugins.sparql.sparql.Query object
  725. """
  726. # We get in: (prologue, query)
  727. prologue = translatePrologue(q[0], base, initNs)
  728. # absolutize/resolve prefixes
  729. q[1] = traverse(
  730. q[1], visitPost=functools.partial(translatePName, prologue=prologue)
  731. )
  732. P, PV = translate(q[1])
  733. datasetClause = q[1].datasetClause
  734. if q[1].name == "ConstructQuery":
  735. template = triples(q[1].template) if q[1].template else None
  736. res = CompValue(q[1].name, p=P, template=template, datasetClause=datasetClause)
  737. else:
  738. res = CompValue(q[1].name, p=P, datasetClause=datasetClause, PV=PV)
  739. res = traverse(res, visitPost=simplify)
  740. _traverseAgg(res, visitor=analyse)
  741. _traverseAgg(res, _addVars)
  742. return Query(prologue, res)
  743. class ExpressionNotCoveredException(Exception): # noqa: N818
  744. pass
  745. class _AlgebraTranslator:
  746. """Translator of a Query's algebra to its equivalent SPARQL (string).
  747. Coded as a class to support storage of state during the translation process,
  748. without use of a file.
  749. Anticipated Usage:
  750. ```python
  751. translated_query = _AlgebraTranslator(query).translateAlgebra()
  752. ```
  753. An external convenience function which wraps the above call,
  754. `translateAlgebra`, is supplied, so this class does not need to be
  755. referenced by client code at all in normal use.
  756. """
  757. def __init__(self, query_algebra: Query):
  758. self.query_algebra = query_algebra
  759. self.aggr_vars: DefaultDict[Identifier, List[Identifier]] = (
  760. collections.defaultdict(list)
  761. )
  762. self._alg_translation: str = ""
  763. def _replace(
  764. self,
  765. old: str,
  766. new: str,
  767. search_from_match: str = None,
  768. search_from_match_occurrence: int = None,
  769. count: int = 1,
  770. ):
  771. def find_nth(haystack, needle, n):
  772. start = haystack.lower().find(needle)
  773. while start >= 0 and n > 1:
  774. start = haystack.lower().find(needle, start + len(needle))
  775. n -= 1
  776. return start
  777. if search_from_match and search_from_match_occurrence:
  778. position = find_nth(
  779. self._alg_translation, search_from_match, search_from_match_occurrence
  780. )
  781. filedata_pre = self._alg_translation[:position]
  782. filedata_post = self._alg_translation[position:].replace(old, new, count)
  783. self._alg_translation = filedata_pre + filedata_post
  784. else:
  785. self._alg_translation = self._alg_translation.replace(old, new, count)
  786. def convert_node_arg(
  787. self, node_arg: typing.Union[Identifier, CompValue, Expr, str]
  788. ) -> str:
  789. if isinstance(node_arg, Identifier):
  790. if node_arg in self.aggr_vars.keys():
  791. grp_var = self.aggr_vars[node_arg].pop(0).n3()
  792. return grp_var
  793. else:
  794. return node_arg.n3()
  795. elif isinstance(node_arg, CompValue):
  796. return "{" + node_arg.name + "}"
  797. elif isinstance(node_arg, str):
  798. return node_arg
  799. else:
  800. raise ExpressionNotCoveredException(
  801. "The expression {0} might not be covered yet.".format(node_arg)
  802. )
  803. def sparql_query_text(self, node):
  804. """<https://www.w3.org/TR/sparql11-query/#sparqlSyntax>"""
  805. if isinstance(node, CompValue):
  806. # 18.2 Query Forms
  807. if node.name == "SelectQuery":
  808. self._alg_translation = "-*-SELECT-*- " + "{" + node.p.name + "}"
  809. # 18.2 Graph Patterns
  810. elif node.name == "BGP":
  811. # Identifiers or Paths
  812. # Negated path throws a type error. Probably n3() method of negated paths should be fixed
  813. triples = "".join(
  814. triple[0].n3() + " " + triple[1].n3() + " " + triple[2].n3() + "."
  815. for triple in node.triples
  816. )
  817. self._replace("{BGP}", triples)
  818. # The dummy -*-SELECT-*- is placed during a SelectQuery or Multiset pattern in order to be able
  819. # to match extended variables in a specific Select-clause (see "Extend" below)
  820. self._replace("-*-SELECT-*-", "SELECT", count=-1)
  821. # If there is no "Group By" clause the placeholder will simply be deleted. Otherwise there will be
  822. # no matching {GroupBy} placeholder because it has already been replaced by "group by variables"
  823. self._replace("{GroupBy}", "", count=-1)
  824. self._replace("{Having}", "", count=-1)
  825. elif node.name == "Join":
  826. self._replace(
  827. "{Join}", "{" + node.p1.name + "}{" + node.p2.name + "}"
  828. ) #
  829. elif node.name == "LeftJoin":
  830. self._replace(
  831. "{LeftJoin}",
  832. "{" + node.p1.name + "}OPTIONAL{{" + node.p2.name + "}}",
  833. )
  834. elif node.name == "Filter":
  835. if isinstance(node.expr, CompValue):
  836. expr = node.expr.name
  837. else:
  838. raise ExpressionNotCoveredException(
  839. "This expression might not be covered yet."
  840. )
  841. if node.p:
  842. # Filter with p=AggregateJoin = Having
  843. if node.p.name == "AggregateJoin":
  844. self._replace("{Filter}", "{" + node.p.name + "}")
  845. self._replace("{Having}", "HAVING({" + expr + "})")
  846. else:
  847. self._replace(
  848. "{Filter}", "FILTER({" + expr + "}) {" + node.p.name + "}"
  849. )
  850. else:
  851. self._replace("{Filter}", "FILTER({" + expr + "})")
  852. elif node.name == "Union":
  853. self._replace(
  854. "{Union}", "{{" + node.p1.name + "}}UNION{{" + node.p2.name + "}}"
  855. )
  856. elif node.name == "Graph":
  857. expr = "GRAPH " + node.term.n3() + " {{" + node.p.name + "}}"
  858. self._replace("{Graph}", expr)
  859. elif node.name == "Extend":
  860. query_string = self._alg_translation.lower()
  861. select_occurrences = query_string.count("-*-select-*-")
  862. self._replace(
  863. node.var.n3(),
  864. "("
  865. + self.convert_node_arg(node.expr)
  866. + " as "
  867. + node.var.n3()
  868. + ")",
  869. search_from_match="-*-select-*-",
  870. search_from_match_occurrence=select_occurrences,
  871. )
  872. self._replace("{Extend}", "{" + node.p.name + "}")
  873. elif node.name == "Minus":
  874. expr = "{" + node.p1.name + "}MINUS{{" + node.p2.name + "}}"
  875. self._replace("{Minus}", expr)
  876. elif node.name == "Group":
  877. group_by_vars = []
  878. if node.expr:
  879. for var in node.expr:
  880. if isinstance(var, Identifier):
  881. group_by_vars.append(var.n3())
  882. else:
  883. raise ExpressionNotCoveredException(
  884. "This expression might not be covered yet."
  885. )
  886. self._replace("{Group}", "{" + node.p.name + "}")
  887. self._replace(
  888. "{GroupBy}", "GROUP BY " + " ".join(group_by_vars) + " "
  889. )
  890. else:
  891. self._replace("{Group}", "{" + node.p.name + "}")
  892. elif node.name == "AggregateJoin":
  893. self._replace("{AggregateJoin}", "{" + node.p.name + "}")
  894. for agg_func in node.A:
  895. if isinstance(agg_func.res, Identifier):
  896. identifier = agg_func.res.n3()
  897. else:
  898. raise ExpressionNotCoveredException(
  899. "This expression might not be covered yet."
  900. )
  901. self.aggr_vars[agg_func.res].append(agg_func.vars)
  902. agg_func_name = agg_func.name.split("_")[1]
  903. distinct = ""
  904. if agg_func.distinct:
  905. distinct = agg_func.distinct + " "
  906. if agg_func_name == "GroupConcat":
  907. self._replace(
  908. identifier,
  909. "GROUP_CONCAT"
  910. + "("
  911. + distinct
  912. + agg_func.vars.n3()
  913. + ";SEPARATOR="
  914. + agg_func.separator.n3()
  915. + ")",
  916. )
  917. else:
  918. self._replace(
  919. identifier,
  920. agg_func_name.upper()
  921. + "("
  922. + distinct
  923. + self.convert_node_arg(agg_func.vars)
  924. + ")",
  925. )
  926. # For non-aggregated variables the aggregation function "sample" is automatically assigned.
  927. # However, we do not want to have "sample" wrapped around non-aggregated variables. That is
  928. # why we replace it. If "sample" is used on purpose it will not be replaced as the alias
  929. # must be different from the variable in this case.
  930. self._replace(
  931. "(SAMPLE({0}) as {0})".format(
  932. self.convert_node_arg(agg_func.vars)
  933. ),
  934. self.convert_node_arg(agg_func.vars),
  935. )
  936. elif node.name == "GroupGraphPatternSub":
  937. self._replace(
  938. "GroupGraphPatternSub",
  939. " ".join([self.convert_node_arg(pattern) for pattern in node.part]),
  940. )
  941. elif node.name == "TriplesBlock":
  942. self._replace(
  943. "{TriplesBlock}",
  944. "".join(
  945. triple[0].n3()
  946. + " "
  947. + triple[1].n3()
  948. + " "
  949. + triple[2].n3()
  950. + "."
  951. for triple in node.triples
  952. ),
  953. )
  954. # 18.2 Solution modifiers
  955. elif node.name == "ToList":
  956. raise ExpressionNotCoveredException(
  957. "This expression might not be covered yet."
  958. )
  959. elif node.name == "OrderBy":
  960. order_conditions = []
  961. for c in node.expr:
  962. if isinstance(c.expr, Identifier):
  963. var = c.expr.n3()
  964. if c.order is not None:
  965. cond = c.order + "(" + var + ")"
  966. else:
  967. cond = var
  968. order_conditions.append(cond)
  969. else:
  970. raise ExpressionNotCoveredException(
  971. "This expression might not be covered yet."
  972. )
  973. self._replace("{OrderBy}", "{" + node.p.name + "}")
  974. self._replace("{OrderConditions}", " ".join(order_conditions) + " ")
  975. elif node.name == "Project":
  976. project_variables = []
  977. for var in node.PV:
  978. if isinstance(var, Identifier):
  979. project_variables.append(var.n3())
  980. else:
  981. raise ExpressionNotCoveredException(
  982. "This expression might not be covered yet."
  983. )
  984. order_by_pattern = ""
  985. if node.p.name == "OrderBy":
  986. order_by_pattern = "ORDER BY {OrderConditions}"
  987. self._replace(
  988. "{Project}",
  989. " ".join(project_variables)
  990. + "{{"
  991. + node.p.name
  992. + "}}"
  993. + "{GroupBy}"
  994. + order_by_pattern
  995. + "{Having}",
  996. )
  997. elif node.name == "Distinct":
  998. self._replace("{Distinct}", "DISTINCT {" + node.p.name + "}")
  999. elif node.name == "Reduced":
  1000. self._replace("{Reduced}", "REDUCED {" + node.p.name + "}")
  1001. elif node.name == "Slice":
  1002. slice = "OFFSET " + str(node.start) + " LIMIT " + str(node.length)
  1003. self._replace("{Slice}", "{" + node.p.name + "}" + slice)
  1004. elif node.name == "ToMultiSet":
  1005. if node.p.name == "values":
  1006. self._replace("{ToMultiSet}", "{{" + node.p.name + "}}")
  1007. else:
  1008. self._replace(
  1009. "{ToMultiSet}", "{-*-SELECT-*- " + "{" + node.p.name + "}" + "}"
  1010. )
  1011. # 18.2 Property Path
  1012. # 17 Expressions and Testing Values
  1013. # # 17.3 Operator Mapping
  1014. elif node.name == "RelationalExpression":
  1015. expr = self.convert_node_arg(node.expr)
  1016. op = node.op
  1017. if isinstance(list, type(node.other)):
  1018. other = (
  1019. "("
  1020. + ", ".join(self.convert_node_arg(expr) for expr in node.other)
  1021. + ")"
  1022. )
  1023. else:
  1024. other = self.convert_node_arg(node.other)
  1025. condition = "{left} {operator} {right}".format(
  1026. left=expr, operator=op, right=other
  1027. )
  1028. self._replace("{RelationalExpression}", condition)
  1029. elif node.name == "ConditionalAndExpression":
  1030. inner_nodes = " && ".join(
  1031. [self.convert_node_arg(expr) for expr in node.other]
  1032. )
  1033. self._replace(
  1034. "{ConditionalAndExpression}",
  1035. self.convert_node_arg(node.expr) + " && " + inner_nodes,
  1036. )
  1037. elif node.name == "ConditionalOrExpression":
  1038. inner_nodes = " || ".join(
  1039. [self.convert_node_arg(expr) for expr in node.other]
  1040. )
  1041. self._replace(
  1042. "{ConditionalOrExpression}",
  1043. "(" + self.convert_node_arg(node.expr) + " || " + inner_nodes + ")",
  1044. )
  1045. elif node.name == "MultiplicativeExpression":
  1046. left_side = self.convert_node_arg(node.expr)
  1047. multiplication = left_side
  1048. for i, operator in enumerate(node.op):
  1049. multiplication += (
  1050. operator + " " + self.convert_node_arg(node.other[i]) + " "
  1051. )
  1052. self._replace("{MultiplicativeExpression}", multiplication)
  1053. elif node.name == "AdditiveExpression":
  1054. left_side = self.convert_node_arg(node.expr)
  1055. addition = left_side
  1056. for i, operator in enumerate(node.op):
  1057. addition += (
  1058. operator + " " + self.convert_node_arg(node.other[i]) + " "
  1059. )
  1060. self._replace("{AdditiveExpression}", addition)
  1061. elif node.name == "UnaryNot":
  1062. self._replace("{UnaryNot}", "!" + self.convert_node_arg(node.expr))
  1063. # # 17.4 Function Definitions
  1064. # # # 17.4.1 Functional Forms
  1065. elif node.name.endswith("BOUND"):
  1066. bound_var = self.convert_node_arg(node.arg)
  1067. self._replace("{Builtin_BOUND}", "bound(" + bound_var + ")")
  1068. elif node.name.endswith("IF"):
  1069. arg2 = self.convert_node_arg(node.arg2)
  1070. arg3 = self.convert_node_arg(node.arg3)
  1071. if_expression = (
  1072. "IF(" + "{" + node.arg1.name + "}, " + arg2 + ", " + arg3 + ")"
  1073. )
  1074. self._replace("{Builtin_IF}", if_expression)
  1075. elif node.name.endswith("COALESCE"):
  1076. self._replace(
  1077. "{Builtin_COALESCE}",
  1078. "COALESCE("
  1079. + ", ".join(self.convert_node_arg(arg) for arg in node.arg)
  1080. + ")",
  1081. )
  1082. elif node.name.endswith("Builtin_EXISTS"):
  1083. # The node's name which we get with node.graph.name returns "Join" instead of GroupGraphPatternSub
  1084. # According to https://www.w3.org/TR/2013/REC-sparql11-query-20130321/#rExistsFunc
  1085. # ExistsFunc can only have a GroupGraphPattern as parameter. However, when we print the query algebra
  1086. # we get a GroupGraphPatternSub
  1087. self._replace(
  1088. "{Builtin_EXISTS}", "EXISTS " + "{{" + node.graph.name + "}}"
  1089. )
  1090. traverse(node.graph, visitPre=self.sparql_query_text)
  1091. return node.graph
  1092. elif node.name.endswith("Builtin_NOTEXISTS"):
  1093. # The node's name which we get with node.graph.name returns "Join" instead of GroupGraphPatternSub
  1094. # According to https://www.w3.org/TR/2013/REC-sparql11-query-20130321/#rNotExistsFunc
  1095. # NotExistsFunc can only have a GroupGraphPattern as parameter. However, when we print the query algebra
  1096. # we get a GroupGraphPatternSub
  1097. self._replace(
  1098. "{Builtin_NOTEXISTS}", "NOT EXISTS " + "{{" + node.graph.name + "}}"
  1099. )
  1100. traverse(node.graph, visitPre=self.sparql_query_text)
  1101. return node.graph
  1102. # # # # 17.4.1.5 logical-or: Covered in "RelationalExpression"
  1103. # # # # 17.4.1.6 logical-and: Covered in "RelationalExpression"
  1104. # # # # 17.4.1.7 RDFterm-equal: Covered in "RelationalExpression"
  1105. elif node.name.endswith("sameTerm"):
  1106. self._replace(
  1107. "{Builtin_sameTerm}",
  1108. "SAMETERM("
  1109. + self.convert_node_arg(node.arg1)
  1110. + ", "
  1111. + self.convert_node_arg(node.arg2)
  1112. + ")",
  1113. )
  1114. # # # # IN: Covered in "RelationalExpression"
  1115. # # # # NOT IN: Covered in "RelationalExpression"
  1116. # # # 17.4.2 Functions on RDF Terms
  1117. elif node.name.endswith("Builtin_isIRI"):
  1118. self._replace(
  1119. "{Builtin_isIRI}", "isIRI(" + self.convert_node_arg(node.arg) + ")"
  1120. )
  1121. elif node.name.endswith("Builtin_isBLANK"):
  1122. self._replace(
  1123. "{Builtin_isBLANK}",
  1124. "isBLANK(" + self.convert_node_arg(node.arg) + ")",
  1125. )
  1126. elif node.name.endswith("Builtin_isLITERAL"):
  1127. self._replace(
  1128. "{Builtin_isLITERAL}",
  1129. "isLITERAL(" + self.convert_node_arg(node.arg) + ")",
  1130. )
  1131. elif node.name.endswith("Builtin_isNUMERIC"):
  1132. self._replace(
  1133. "{Builtin_isNUMERIC}",
  1134. "isNUMERIC(" + self.convert_node_arg(node.arg) + ")",
  1135. )
  1136. elif node.name.endswith("Builtin_STR"):
  1137. self._replace(
  1138. "{Builtin_STR}", "STR(" + self.convert_node_arg(node.arg) + ")"
  1139. )
  1140. elif node.name.endswith("Builtin_LANG"):
  1141. self._replace(
  1142. "{Builtin_LANG}", "LANG(" + self.convert_node_arg(node.arg) + ")"
  1143. )
  1144. elif node.name.endswith("Builtin_DATATYPE"):
  1145. self._replace(
  1146. "{Builtin_DATATYPE}",
  1147. "DATATYPE(" + self.convert_node_arg(node.arg) + ")",
  1148. )
  1149. elif node.name.endswith("Builtin_IRI"):
  1150. self._replace(
  1151. "{Builtin_IRI}", "IRI(" + self.convert_node_arg(node.arg) + ")"
  1152. )
  1153. elif node.name.endswith("Builtin_BNODE"):
  1154. self._replace(
  1155. "{Builtin_BNODE}", "BNODE(" + self.convert_node_arg(node.arg) + ")"
  1156. )
  1157. elif node.name.endswith("STRDT"):
  1158. self._replace(
  1159. "{Builtin_STRDT}",
  1160. "STRDT("
  1161. + self.convert_node_arg(node.arg1)
  1162. + ", "
  1163. + self.convert_node_arg(node.arg2)
  1164. + ")",
  1165. )
  1166. elif node.name.endswith("Builtin_STRLANG"):
  1167. self._replace(
  1168. "{Builtin_STRLANG}",
  1169. "STRLANG("
  1170. + self.convert_node_arg(node.arg1)
  1171. + ", "
  1172. + self.convert_node_arg(node.arg2)
  1173. + ")",
  1174. )
  1175. elif node.name.endswith("Builtin_UUID"):
  1176. self._replace("{Builtin_UUID}", "UUID()")
  1177. elif node.name.endswith("Builtin_STRUUID"):
  1178. self._replace("{Builtin_STRUUID}", "STRUUID()")
  1179. # # # 17.4.3 Functions on Strings
  1180. elif node.name.endswith("Builtin_STRLEN"):
  1181. self._replace(
  1182. "{Builtin_STRLEN}",
  1183. "STRLEN(" + self.convert_node_arg(node.arg) + ")",
  1184. )
  1185. elif node.name.endswith("Builtin_SUBSTR"):
  1186. args = [self.convert_node_arg(node.arg), node.start]
  1187. if node.length:
  1188. args.append(node.length)
  1189. expr = "SUBSTR(" + ", ".join(args) + ")"
  1190. self._replace("{Builtin_SUBSTR}", expr)
  1191. elif node.name.endswith("Builtin_UCASE"):
  1192. self._replace(
  1193. "{Builtin_UCASE}", "UCASE(" + self.convert_node_arg(node.arg) + ")"
  1194. )
  1195. elif node.name.endswith("Builtin_LCASE"):
  1196. self._replace(
  1197. "{Builtin_LCASE}", "LCASE(" + self.convert_node_arg(node.arg) + ")"
  1198. )
  1199. elif node.name.endswith("Builtin_STRSTARTS"):
  1200. self._replace(
  1201. "{Builtin_STRSTARTS}",
  1202. "STRSTARTS("
  1203. + self.convert_node_arg(node.arg1)
  1204. + ", "
  1205. + self.convert_node_arg(node.arg2)
  1206. + ")",
  1207. )
  1208. elif node.name.endswith("Builtin_STRENDS"):
  1209. self._replace(
  1210. "{Builtin_STRENDS}",
  1211. "STRENDS("
  1212. + self.convert_node_arg(node.arg1)
  1213. + ", "
  1214. + self.convert_node_arg(node.arg2)
  1215. + ")",
  1216. )
  1217. elif node.name.endswith("Builtin_CONTAINS"):
  1218. self._replace(
  1219. "{Builtin_CONTAINS}",
  1220. "CONTAINS("
  1221. + self.convert_node_arg(node.arg1)
  1222. + ", "
  1223. + self.convert_node_arg(node.arg2)
  1224. + ")",
  1225. )
  1226. elif node.name.endswith("Builtin_STRBEFORE"):
  1227. self._replace(
  1228. "{Builtin_STRBEFORE}",
  1229. "STRBEFORE("
  1230. + self.convert_node_arg(node.arg1)
  1231. + ", "
  1232. + self.convert_node_arg(node.arg2)
  1233. + ")",
  1234. )
  1235. elif node.name.endswith("Builtin_STRAFTER"):
  1236. self._replace(
  1237. "{Builtin_STRAFTER}",
  1238. "STRAFTER("
  1239. + self.convert_node_arg(node.arg1)
  1240. + ", "
  1241. + self.convert_node_arg(node.arg2)
  1242. + ")",
  1243. )
  1244. elif node.name.endswith("Builtin_ENCODE_FOR_URI"):
  1245. self._replace(
  1246. "{Builtin_ENCODE_FOR_URI}",
  1247. "ENCODE_FOR_URI(" + self.convert_node_arg(node.arg) + ")",
  1248. )
  1249. elif node.name.endswith("Builtin_CONCAT"):
  1250. expr = "CONCAT({vars})".format(
  1251. vars=", ".join(self.convert_node_arg(elem) for elem in node.arg)
  1252. )
  1253. self._replace("{Builtin_CONCAT}", expr)
  1254. elif node.name.endswith("Builtin_LANGMATCHES"):
  1255. self._replace(
  1256. "{Builtin_LANGMATCHES}",
  1257. "LANGMATCHES("
  1258. + self.convert_node_arg(node.arg1)
  1259. + ", "
  1260. + self.convert_node_arg(node.arg2)
  1261. + ")",
  1262. )
  1263. elif node.name.endswith("REGEX"):
  1264. args = [
  1265. self.convert_node_arg(node.text),
  1266. self.convert_node_arg(node.pattern),
  1267. ]
  1268. expr = "REGEX(" + ", ".join(args) + ")"
  1269. self._replace("{Builtin_REGEX}", expr)
  1270. elif node.name.endswith("REPLACE"):
  1271. self._replace(
  1272. "{Builtin_REPLACE}",
  1273. "REPLACE("
  1274. + self.convert_node_arg(node.arg)
  1275. + ", "
  1276. + self.convert_node_arg(node.pattern)
  1277. + ", "
  1278. + self.convert_node_arg(node.replacement)
  1279. + ")",
  1280. )
  1281. # # # 17.4.4 Functions on Numerics
  1282. elif node.name == "Builtin_ABS":
  1283. self._replace(
  1284. "{Builtin_ABS}", "ABS(" + self.convert_node_arg(node.arg) + ")"
  1285. )
  1286. elif node.name == "Builtin_ROUND":
  1287. self._replace(
  1288. "{Builtin_ROUND}", "ROUND(" + self.convert_node_arg(node.arg) + ")"
  1289. )
  1290. elif node.name == "Builtin_CEIL":
  1291. self._replace(
  1292. "{Builtin_CEIL}", "CEIL(" + self.convert_node_arg(node.arg) + ")"
  1293. )
  1294. elif node.name == "Builtin_FLOOR":
  1295. self._replace(
  1296. "{Builtin_FLOOR}", "FLOOR(" + self.convert_node_arg(node.arg) + ")"
  1297. )
  1298. elif node.name == "Builtin_RAND":
  1299. self._replace("{Builtin_RAND}", "RAND()")
  1300. # # # 17.4.5 Functions on Dates and Times
  1301. elif node.name == "Builtin_NOW":
  1302. self._replace("{Builtin_NOW}", "NOW()")
  1303. elif node.name == "Builtin_YEAR":
  1304. self._replace(
  1305. "{Builtin_YEAR}", "YEAR(" + self.convert_node_arg(node.arg) + ")"
  1306. )
  1307. elif node.name == "Builtin_MONTH":
  1308. self._replace(
  1309. "{Builtin_MONTH}", "MONTH(" + self.convert_node_arg(node.arg) + ")"
  1310. )
  1311. elif node.name == "Builtin_DAY":
  1312. self._replace(
  1313. "{Builtin_DAY}", "DAY(" + self.convert_node_arg(node.arg) + ")"
  1314. )
  1315. elif node.name == "Builtin_HOURS":
  1316. self._replace(
  1317. "{Builtin_HOURS}", "HOURS(" + self.convert_node_arg(node.arg) + ")"
  1318. )
  1319. elif node.name == "Builtin_MINUTES":
  1320. self._replace(
  1321. "{Builtin_MINUTES}",
  1322. "MINUTES(" + self.convert_node_arg(node.arg) + ")",
  1323. )
  1324. elif node.name == "Builtin_SECONDS":
  1325. self._replace(
  1326. "{Builtin_SECONDS}",
  1327. "SECONDS(" + self.convert_node_arg(node.arg) + ")",
  1328. )
  1329. elif node.name == "Builtin_TIMEZONE":
  1330. self._replace(
  1331. "{Builtin_TIMEZONE}",
  1332. "TIMEZONE(" + self.convert_node_arg(node.arg) + ")",
  1333. )
  1334. elif node.name == "Builtin_TZ":
  1335. self._replace(
  1336. "{Builtin_TZ}", "TZ(" + self.convert_node_arg(node.arg) + ")"
  1337. )
  1338. # # # 17.4.6 Hash functions
  1339. elif node.name == "Builtin_MD5":
  1340. self._replace(
  1341. "{Builtin_MD5}", "MD5(" + self.convert_node_arg(node.arg) + ")"
  1342. )
  1343. elif node.name == "Builtin_SHA1":
  1344. self._replace(
  1345. "{Builtin_SHA1}", "SHA1(" + self.convert_node_arg(node.arg) + ")"
  1346. )
  1347. elif node.name == "Builtin_SHA256":
  1348. self._replace(
  1349. "{Builtin_SHA256}",
  1350. "SHA256(" + self.convert_node_arg(node.arg) + ")",
  1351. )
  1352. elif node.name == "Builtin_SHA384":
  1353. self._replace(
  1354. "{Builtin_SHA384}",
  1355. "SHA384(" + self.convert_node_arg(node.arg) + ")",
  1356. )
  1357. elif node.name == "Builtin_SHA512":
  1358. self._replace(
  1359. "{Builtin_SHA512}",
  1360. "SHA512(" + self.convert_node_arg(node.arg) + ")",
  1361. )
  1362. # Other
  1363. elif node.name == "values":
  1364. columns = []
  1365. for key in node.res[0].keys():
  1366. if isinstance(key, Identifier):
  1367. columns.append(key.n3())
  1368. else:
  1369. raise ExpressionNotCoveredException(
  1370. "The expression {0} might not be covered yet.".format(key)
  1371. )
  1372. values = "VALUES (" + " ".join(columns) + ")"
  1373. rows = ""
  1374. for elem in node.res:
  1375. row = []
  1376. for term in elem.values():
  1377. if isinstance(term, Identifier):
  1378. row.append(
  1379. term.n3()
  1380. ) # n3() is not part of Identifier class but every subclass has it
  1381. elif isinstance(term, str):
  1382. row.append(term)
  1383. else:
  1384. raise ExpressionNotCoveredException(
  1385. "The expression {0} might not be covered yet.".format(
  1386. term
  1387. )
  1388. )
  1389. rows += "(" + " ".join(row) + ")"
  1390. self._replace("values", values + "{" + rows + "}")
  1391. elif node.name == "ServiceGraphPattern":
  1392. self._replace(
  1393. "{ServiceGraphPattern}",
  1394. "SERVICE "
  1395. + self.convert_node_arg(node.term)
  1396. + "{"
  1397. + node.graph.name
  1398. + "}",
  1399. )
  1400. traverse(node.graph, visitPre=self.sparql_query_text)
  1401. return node.graph
  1402. # else:
  1403. # raise ExpressionNotCoveredException("The expression {0} might not be covered yet.".format(node.name))
  1404. def translateAlgebra(self) -> str:
  1405. traverse(self.query_algebra.algebra, visitPre=self.sparql_query_text)
  1406. return self._alg_translation
  1407. def translateAlgebra(query_algebra: Query) -> str:
  1408. """
  1409. Translates a SPARQL 1.1 algebra tree into the corresponding query string.
  1410. Args:
  1411. query_algebra: An algebra returned by `translateQuery`.
  1412. Returns:
  1413. The query form generated from the SPARQL 1.1 algebra tree for
  1414. SELECT queries.
  1415. """
  1416. query_from_algebra = _AlgebraTranslator(
  1417. query_algebra=query_algebra
  1418. ).translateAlgebra()
  1419. return query_from_algebra
  1420. def pprintAlgebra(q) -> None:
  1421. def pp(p, ind=" "):
  1422. # if isinstance(p, list):
  1423. # print "[ "
  1424. # for x in p: pp(x,ind)
  1425. # print "%s ]"%ind
  1426. # return
  1427. if not isinstance(p, CompValue):
  1428. print(p)
  1429. return
  1430. print("%s(" % (p.name,))
  1431. for k in p:
  1432. print(
  1433. "%s%s ="
  1434. % (
  1435. ind,
  1436. k,
  1437. ),
  1438. end=" ",
  1439. )
  1440. pp(p[k], ind + " ")
  1441. print("%s)" % ind)
  1442. try:
  1443. pp(q.algebra)
  1444. except AttributeError:
  1445. # it's update, just a list
  1446. for x in q:
  1447. pp(x)