diff --git a/.gitignore b/.gitignore index a5810f6..014fdc3 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ share/python-wheels/ .installed.cfg *.egg MANIFEST +.DS_Store # PyInstaller # Usually these files are written by a python script from a template @@ -98,7 +99,7 @@ ipython_config.py # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. # This is especially recommended for binary packages to ensure reproducibility, and is more # commonly ignored for libraries. -#uv.lock +uv.lock # poetry # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. @@ -173,7 +174,7 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.idea/ # Abstra # Abstra is an AI-powered process automation framework. diff --git a/baseline_quest/README.md b/baseline_quest/README.md new file mode 100644 index 0000000..df3daff --- /dev/null +++ b/baseline_quest/README.md @@ -0,0 +1,45 @@ +# Baseline Experiments with QUEST + +## Experiment Setup +1. Indexed with Chroma (`index_documents.py`) + - 512 tokens with 80-token overlap / index only first 512 tokens (same as QUEST) + - Embedded in batches of 256 chunks + - Embedding model: `bge-small-en-v1.5` + - Note: `index_documents.py` creates the collection if specified collection in `config.yaml` does not exist. + +2. Decompose (optional) + - Use gpt-4o-mini to decompose the query into subqueries connected with operators + - E.g. "Stoloniferous plants or crops originating from Bolivia" -> retrieve("crops from Bolivia", 100) | retrieve("stoloniferous plants", 100) + - E.g. "Neogene mammals of Africa that are Odd-toed ungulates" -> retrieve("Neogene mammals of Africa", 100) & retrieve("Odd-toed ungulates", 100) + - Examples decomposition python files: `examples/query_9` and `examples/query_10` + - Generate decompositions with `decompose_retrieve.py --mode generate` + +3. Retrieval (@ k) + - Retrieve with query / subquery (vector similarity) + - If indexed entired documented, retrieves the top 200 most likely chunks, then maps them to the document. + - If indexed first 512 tokens only, retrieves the top k most likely chunks. + - `semantic_retrieval.py` + - Retrieve after decomposition with query / subquery + - `decompose_retrieve.py --mode execute` executes all of the generated decomposition pythons scripts from step 2 and uses the same vectory similarity retrieval for each subquery. + +4. Calculate Recall@K + - For semantic retrieval, this step is done with the script `semantic_retrieval.py`. + - For decompose and retrieve, run `decompose_retrieve.py --mode analyze`. + +## Data +The data is directly from QUEST (https://github.com/google-research/language/tree/master/language/quest#examples). +- The documents that are embedded are: https://storage.googleapis.com/gresearch/quest/documents.zip +- `data/train_subset1.jsonl` is 20 randomly sampled queries from `train.jsonl` of QUEST. +- `data/train_subset2.jsonl` is 20 randomly sampled non-union queries from `train.jsonl` of QUEST. + + + +## Results: +| | Retrieve (entire document) | Retrieve (first 512 tokens) | Decompose + Retrieve* (entire document) | Decompose + Retrieve* (first 512 tokens) | +|--------------|----------------------------|-----------------------------|-----------------------------------------------------------------|-----------------------------------------------------------------| +| Recall @ 20 | 0.0886 | 0.1127 | - | | +| Recall @ 50 | 0.1663 | 0.1593 | 0.1560 (\|Pred\| = 61.10) | 0.1617 (\|Pred\| = 60.70) | +| Recall @ 100 | 0.2122 | 0.2250 | 0.2285 (\|Pred\| = 205.95) (k for subqueries increased in size) | 0.2157 (\|Pred\| = 209.30) (k for subqueries increased in size) | diff --git a/baseline_quest/config.yaml b/baseline_quest/config.yaml new file mode 100644 index 0000000..9be7122 --- /dev/null +++ b/baseline_quest/config.yaml @@ -0,0 +1,22 @@ +indexing: + embedding_model: "BAAI/bge-small-en-v1.5" + index_first_512: True + chunk_size: 512 + overlap: 80 + batch_size: 512 + + chroma: + persist_dir: "./chroma_quest" + collection: "quest_documents" + +data: + document_path: /orcd/home/002/joycequ/quest_data/documents.jsonl + queries_file: data/train_subset1.jsonl # gold path + +retrieval: + top_k: 100 + include_chunks: True + +decomposition: + top_k: 100 + llm_model: gpt-4o-mini \ No newline at end of file diff --git a/baseline_quest/data/train_subset1.jsonl b/baseline_quest/data/train_subset1.jsonl new file mode 100644 index 0000000..78725c6 --- /dev/null +++ b/baseline_quest/data/train_subset1.jsonl @@ -0,0 +1,20 @@ +{"query": "Stoloniferous plants or crops originating from Bolivia", "docs": ["Ullucus", "Darlingtonia californica", "Oxalis tuberosa", "Fragaria", "Pachysandra procumbens", "Pachysandra terminalis", "Chimaphila umbellata", "Premnotrypes", "Mentha", "Ficinia spiralis", "Nicotiana glauca", "Tamarillo", "Tropaeolum tuberosum", "Potato", "Vinca", "Chimaphila maculata", "Mirabilis expansa", "Hedera", "Brazil nut", "Colorado potato beetle"], "original_query": "Stoloniferous plants or Crops originating from Bolivia", "scores": null, "metadata": {"template": "_ or _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "plants"}} +{"query": "what are Novels by Robert B. Parker that are not set in Massachusetts", "docs": ["Split Image (novel)", "Stranger in Paradise (novel)", "The Boxer and the Spy", "Sea Change (Parker novel)", "Night and Day (Parker novel)", "High Profile", "Love and Glory", "Death in Paradise (novel)", "Perchance to Dream (novel)", "Blue Screen (novel)", "Appaloosa (novel)", "Wilderness (Parker novel)", "Stone Cold (Parker novel)"], "original_query": "Novels by Robert B. Parker that are not Novels set in Massachusetts", "scores": null, "metadata": {"template": "_ that are not _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Split Image (novel)": ["Definitely relevant"], "Stranger in Paradise (novel)": ["Likely relevant"], "The Boxer and the Spy": ["Definitely relevant"], "Sea Change (Parker novel)": ["Definitely relevant"], "Night and Day (Parker novel)": ["Definitely relevant"], "High Profile": ["Definitely relevant"], "Love and Glory": ["Definitely relevant"], "Death in Paradise (novel)": ["Definitely relevant"], "Perchance to Dream (novel)": ["Definitely relevant"], "Blue Screen (novel)": ["Definitely relevant"], "Appaloosa (novel)": ["Definitely relevant"], "Wilderness (Parker novel)": ["Likely relevant"], "Stone Cold (Parker novel)": ["Definitely relevant"]}, "evidence_ratings": {"Split Image (novel)": ["Complete"], "Stranger in Paradise (novel)": ["Partial"], "The Boxer and the Spy": ["Complete"], "Sea Change (Parker novel)": ["Complete"], "Night and Day (Parker novel)": ["Partial"], "High Profile": ["Partial"], "Love and Glory": ["Complete"], "Death in Paradise (novel)": ["Complete"], "Perchance to Dream (novel)": ["Complete"], "Blue Screen (novel)": ["Complete"], "Appaloosa (novel)": ["Complete"], "Wilderness (Parker novel)": ["Partial"], "Stone Cold (Parker novel)": ["Complete"]}, "attributions": {"Split Image (novel)": [{"Novels by Robert B. Parker": "'''''Split Image''''' is a crime novel by Robert B. Parker"}], "Stranger in Paradise (novel)": [{"Novels by Robert B. Parker": "'''''Stranger in Paradise''''' is a 2008 crime novel by Robert B. Parker, the seventh in his Jesse Stone series.\nIn ''Stranger in Paradise'', Wilson \"Crow\" Cromartie from ''Trouble in Paradise'' returns to the quiet town. On arrival he meets with Jesse Stone to let him know he\u2019s in town looking for someone. Unfortunately Jesse cannot arrest him because the statute of limitations has run out for the Stiles Island robbery Crow was involved in ten years earlier that cost residents over $20 million in cash. And since Jesse has no evidence linking him to any of the murders, he has no choice but to let Crow go about his business."}], "The Boxer and the Spy": [{"Novels by Robert B. Parker": "'''''The Boxer and the Spy''''' (2008) is a crime novel for young adults by American author Robert B. Parker."}], "Sea Change (Parker novel)": [{"what are Novels by Robert B. Parker": "is a 2006 crime novel by Robert B. Parker, the fifth in his Jesse Stone series.", "that are not set in Massachusetts": "The woman turns out to be Florence \u201cFlo\u201d Horvath, of Miami, Florida."}], "Night and Day (Parker novel)": [{"what are Novels by Robert B. Parker": "'''''Night and Day''''' is a crime novel by Robert B. Parker,"}], "High Profile": [{"what are Novels by Robert B. Parker that are not set in Massachusetts": "'''''High Profile''''' is a crime novel by Robert B. Parker, the sixth in his Jesse Stone series."}], "Love and Glory": [{"Novels by Robert B. Parker": "'''''Love and Glory''''' () is a 1983 novel by Robert B. Parker.", "not set in Massachusetts": "Much of the novel takes place at the fictional Taft University, which Parker used later as the setting for the 1989 Spenser novel ''Playmates'' and where Susan Silverman teaches."}], "Death in Paradise (novel)": [{"Novels by Robert B. Parker": "'''Death in Paradise''''' is a crime novel by Robert B. Parker"}], "Perchance to Dream (novel)": [{"Novels by Robert B. Parker": "'''''Perchance to Dream''''' is a detective crime novel by Robert B. Parker,"}], "Blue Screen (novel)": [{"what are Novels by Robert B. Parker that are not set in Massachusetts": "'''''Blue Screen''''' is a crime novel by Robert B. Parker, the fifth in his Sunny Randall series.\nSunny Randall is approached by Buddy Bollen to provide protection for his number one client, Erin Flint, star of the Woman Warrior movie series and future star of Bollen\u2019s major league baseball team.. Bollen\u2019s fears prove well founded when Erin\u2019s assistant, Misty, is murdered. Because of Misty\u2019s striking resemblance to her, Erin is convinced the killer was after her. Sunny meets Paradise Police Chief Jesse Stone at the scene of the crime; however Buddy and Erin lack confidence in the Paradise police, and ask Sunny to solve the crime.\n\n\nSunny talks to a sports writer who is convinced that Erin Flint\u2019s addition to Bollen\u2019s baseball team is a publicity stunt and that Erin will be able to compete with the male players in the major league. He proves to be right later when Erin faces a major league pitcher and cannot hit one ball.\n\n\nSunny discovers that Erin Flint is actually Ethel Boverini, and that she is still married to pimp Gerard Basgall. Sunny and an LAPD detective go to question Basgall, who admits to still being married to Erin and still loving her. Erin admits that she was one of Basgall\u2019s prostitutes, as well as his wife. She and Misty Tyler, who is really her sister Edith, began working for Basgall after their mother died."}], "Appaloosa (novel)": [{"what are Novels by Robert B. Parker that are not set in Massachusetts": "'''''Appaloosa''''' (2005) is a novel set in the American Old West written by Robert B. Parker. A film of the same name based on the novel was released in 2008. Parker published a sequel, ''Resolution'', in June 2008 and a third novel featuring the characters of Virgil Cole and Everett Hitch, ''Brimstone'', in May 2009. A fourth and final book in the series\u2014''Blue-Eyed Devil''\u2014was published in 2010 shortly before Parker's death."}], "Wilderness (Parker novel)": [{"Novels by Robert B. Parker": "''Wilderness''' is a novel by American writer Robert B. Parker."}], "Stone Cold (Parker novel)": [{"Novels by Robert B. Parker": "'''''Stone Cold''''' is a crime novel by Robert B. Parker, the fourth in his Jesse Stone series."}]}}} +{"query": "Endemic flora of Australia, Malaysia,and Fiji", "docs": ["Calanthe triplicata", "Phlegmariurus phlegmaria"], "original_query": "Endemic flora of Australia that are also both Flora of Malaysia and Flora of Fiji", "scores": null, "metadata": {"template": "_ that are also both _ and _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "plants"}} +{"query": "Aquatic animals from South America that are found in Victoria(Australia)", "docs": ["Southern elephant seal", "Crabeater seal", "Subantarctic fur seal"], "original_query": "Mammals of South America that are also both Aquatic mammals and Fauna of Victoria (Australia)", "scores": null, "metadata": {"template": "_ that are also both _ and _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "animals"}} +{"query": "Holarctic and North American desert fauna and also Vertebrates of Belize", "docs": ["Couch's kingbird", "Brown jay", "Olive sparrow", "Plain chachalaca", "Buff-bellied hummingbird", "Morelet's seedeater", "Rio Grande leopard frog", "Green jay", "Leptodactylus fragilis", "Hooded oriole"], "original_query": "North American desert fauna that are also both Vertebrates of Belize and Holarctic fauna", "scores": null, "metadata": {"template": "_ that are also both _ and _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "animals"}} +{"query": "Arecaceae that are trees of Indo-China", "docs": ["Coconut", "Eleiodoxa"], "original_query": "Arecaceae that are also Trees of Indo-China", "scores": null, "metadata": {"template": "_ that are also _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "plants"}} +{"query": "Pinnipeds of Antarctica or South America or Marine fauna of Antarctica", "docs": ["Kawas (genus)", "Crabeater seal", "Antarctic fur seal", "Southern elephant seal", "Gal\u00e1pagos sea lion", "Gal\u00e1pagos fur seal", "South American fur seal", "Ross seal", "Leopard seal", "Caribbean monk seal", "Anoxycalyx joubini", "Hadrokirus", "South American sea lion", "Subantarctic fur seal", "Weddell seal", "Juan Fern\u00e1ndez fur seal", "Cirroctopus mawsoni", "Acrophoca", "Piscophoca"], "original_query": "Pinnipeds of Antarctica or Marine fauna of Antarctica or Pinnipeds of South America", "scores": null, "metadata": {"template": "_ or _ or _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "animals"}} +{"query": "find me, Cisuralian animals, Paleozoic insects of Asia, or Carboniferous animals of Asia.", "docs": ["Neostreptognathodus", "Dombarites", "Strigulla", "Thaiaspis", "Palaeontinoidea", "Scutistriga", "Petalodus", "Silverfish", "Sandalodus", "Auchenorrhyncha", "Tyulkinia", "Longzhua", "Unionopterus", "Lysorophia"], "original_query": "Cisuralian animals or Paleozoic insects of Asia or Carboniferous animals of Asia", "scores": null, "metadata": {"template": "_ or _ or _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "animals"}} +{"query": "romance films from New Zealand", "docs": ["My Wedding and Other Secrets", "The Piano", "The Breaker Upperers", "Eagle vs Shark", "The Light Between Oceans (film)", "Bonjour Timothy", "A Soldier's Tale"], "original_query": "New Zealand romance films", "scores": null, "metadata": {"template": "_", "domain": "films", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"My Wedding and Other Secrets": ["Likely relevant"], "The Piano": ["Definitely relevant"], "The Breaker Upperers": ["Definitely relevant"], "Eagle vs Shark": ["Definitely relevant"], "The Light Between Oceans (film)": ["Definitely relevant"], "Bonjour Timothy": ["Definitely relevant"], "A Soldier's Tale": ["Definitely relevant"]}, "evidence_ratings": {"My Wedding and Other Secrets": ["Partial"], "The Piano": ["Partial"], "The Breaker Upperers": ["Complete"], "Eagle vs Shark": ["Complete"], "The Light Between Oceans (film)": ["Complete"], "Bonjour Timothy": ["Complete"], "A Soldier's Tale": ["Complete"]}, "attributions": {"My Wedding and Other Secrets": [{"romance films from New Zealand": "A Romeo and Juliet story set in Auckland, New Zealand, Emily Chu (Michelle Ang) is the daughter of traditional Chinese parents, whose only wishes are that she marries a good Chinese boy and becomes a doctor. But life seems to have other ideas for Emily, who dreams of becoming a world-famous director and falls in love with a white boy from university, James Harrison (Matt Whelan)."}], "The Piano": [{"romance films from New Zealand": "'''''The Piano''''' is a 1993 period drama film written and directed by Jane Campion. Starring Holly Hunter, Harvey Keitel, Sam Neill, and Anna Paquin in her first major acting role, the film focuses on a mute Scottish woman who travels to a remote part of New Zealand"}], "The Breaker Upperers": [{"romance films from New Zealand": "'''''The Breaker Upperers''''' is a New Zealand romantic-comedy film"}], "Eagle vs Shark": [{"romance films from New Zealand": "'''''Eagle vs Shark''''' is a 2007 New Zealand romantic comedy film written and directed by Taika Waititi (in his directorial debut) and financed by the New Zealand Film Commission."}], "The Light Between Oceans (film)": [{"romance films from New Zealand": "'''''The Light Between Oceans''''' is a 2016 romantic drama film written and directed by Derek Cianfrance and based on the 2012 novel of the same name by M. L. Stedman. An international co-production between the United States, Australia, the United Kingdom, and New Zealand, the film stars Michael Fassbender, Alicia Vikander, Rachel Weisz, Bryan Brown, and Jack Thompson. The film tells the story of a lighthouse keeper and his wife who rescue and adopt an infant girl adrift at sea. Years later, the couple discover the child's true parentage and are faced with the moral dilemma of their actions."}], "Bonjour Timothy": [{"romance films from New Zealand": "'''''Bonjour Timothy''''' is a 1995 Canadian-New Zealander romantic comedy film"}], "A Soldier's Tale": [{"romance films from New Zealand": "''''A Soldier's Tale''''' is a 1988 New Zealand romantic war film directed and produced by Larry Parr and starring Gabriel Byrne and Marianne Basler."}]}}} +{"query": "1912 films set in England", "docs": ["A Canine Sherlock Holmes", "The Poacher's Pardon", "Peg Woffington (1912 film)", "Nicholas Nickleby (1912 film)", "Santa Claus (1912 film)", "Oliver Twist (1912 British film)", "Les Amours de la reine \u00c9lisabeth", "A Cornish Romance", "The Charge of the Light Brigade (1912 film)", "Lorna Doone (1912 film)", "Marie Tudor (1912 film)"], "original_query": "Films set in England that are also 1912 films", "scores": null, "metadata": {"template": "_ that are also _", "domain": "films", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"A Canine Sherlock Holmes": ["Likely relevant"], "The Poacher's Pardon": ["Definitely relevant"], "Peg Woffington (1912 film)": ["Likely relevant"], "Nicholas Nickleby (1912 film)": ["Likely relevant"], "Santa Claus (1912 film)": ["Likely relevant"], "Oliver Twist (1912 British film)": ["Likely relevant"], "Les Amours de la reine \u00c9lisabeth": ["Definitely relevant"], "A Cornish Romance": ["Likely relevant"], "The Charge of the Light Brigade (1912 film)": ["Likely relevant"], "Lorna Doone (1912 film)": ["Definitely relevant"], "Marie Tudor (1912 film)": ["Definitely relevant"]}, "evidence_ratings": {"A Canine Sherlock Holmes": ["Partial"], "The Poacher's Pardon": ["Complete"], "Peg Woffington (1912 film)": ["Partial"], "Nicholas Nickleby (1912 film)": ["Partial"], "Santa Claus (1912 film)": ["Complete"], "Oliver Twist (1912 British film)": ["Partial"], "Les Amours de la reine \u00c9lisabeth": ["Complete"], "A Cornish Romance": ["Partial"], "The Charge of the Light Brigade (1912 film)": ["Partial"], "Lorna Doone (1912 film)": ["Complete"], "Marie Tudor (1912 film)": ["Complete"]}, "attributions": {"A Canine Sherlock Holmes": [{"1912 films": "'''''A Canine Sherlock Holmes''''' is a 1912 English silent short film crime drama"}], "The Poacher's Pardon": [{"set in England": "The film was shot in England.", "1912 films": "a 1912 American silent film"}], "Peg Woffington (1912 film)": [{"1912 films set in England": "'''''Peg Woffington''''' is a 1912 British silent historical film directed by A. E. Coleby and starring Leslie Howard Gordon. The film is based on the 1852 play ''Masks and Faces'' by Tom Taylor and Charles Reade. The play had previously been turned into a 1910 American film, and several further adaptations followed. It features the eighteenth century Irish actress Peg Woffington as a major character."}], "Nicholas Nickleby (1912 film)": [{"1912 films set in England": "'''''Nicholas Nickleby''''' is a 1912 American silent short drama film directed by George Nichols, adapted from Charles Dickens' 1839 novel of the same name. The two-reel film stars Harry Benham in the title role and Mignon Anderson.\n\n\n''Nicholas Nickleby'' was produced by the Thanhouser Company, which had previously produced versions of Dickens' ''David Copperfield'' and ''The Old Curiosity Shop'' in 1911.\nAs a 30-minute silent production (some sources state 20 minutes), the film has to significantly compact and truncate the story of the 900 plus page novel. Many characters are not directly identified by name, but would be familiar to those with knowledge of the novel."}], "Santa Claus (1912 film)": [{"1912 films": "a 1912 fantasy silent film", "set in England": "The film was based on a stage play of the same name that played with the same cast at the Scala Theatre in London where it ran from 23 December 1912 to 8 January 1913 with a score by Theodore Holland. A suite by Holland based on music from the play was frequently broadcast at Christmas time during the 1920s and 1930s on BBC Radio. The play was a combination of live-action and two-tone Kinemacolor film. In the UK the film was distributed by the Natural Colour Kinematograph Co. in 1912"}], "Oliver Twist (1912 British film)": [{"1912 films set in England": "'''''Oliver Twist''''' is a 1912 British silent drama film directed by Thomas Bentley and starring Ivy Millais, Alma Taylor and Harry Royston. It is an adaptation of the 1838 novel ''Oliver Twist'' by Charles Dickens. It was the directorial debut of Bentley who went on to become a leading British director. It was the first in a series of Dickens adaptations by Bentley.\n* Ivy Millais as Oliver Twist"}], "Les Amours de la reine \u00c9lisabeth": [{"1912 films set in England": "'''''Les Amours de la reine \u00c9lisabeth''''' (''The Loves of Queen Elizabeth''), '''''Les Amours d'\u00c9lisabeth, Reine d'Angleterre''''' (''The Loves of Elizabeth, Queen of England'') or '''''La reine \u00c9lisabeth''''' (''Queen Elizabeth'') is a 1912 feature 4-reel French silent film based on the love affair between Elizabeth I of England and the Earl of Essex"}], "A Cornish Romance": [{"1912 films": "'''''A Cornish Romance''''' is a 1912 British silent romance film directed by Sidney Northcote and starring Wallett Waller, Dorothy Foster and O'Neil Farrell."}], "The Charge of the Light Brigade (1912 film)": [{"1912 films": "'''''The Charge of the Light Brigade''''' is a 1912 American silent historical drama film directed by J. Searle Dawley."}], "Lorna Doone (1912 film)": [{"1912 films set in England": "'''''Lorna Doone''''' is a 1912 British silent historical film directed by Wilfred Noy and starring Dorothy Bellew. The film is an adaptation of the 1869 novel ''Lorna Doone'' by R. D. Blackmore, set in Seventeenth century Devon."}], "Marie Tudor (1912 film)": [{"1912 films": "''''Marie Tudor''''' is a 1912 French silent historical film directed by Albert Capellani and starring Jeanne Delvair, Paul Capellani and Romuald Joub\u00e9.", "set in England": "The film is an adaptation of Victor Hugo's 1833 play of the same title set at the court of Mary I of England."}]}}} +{"query": "American action films about security and surveillance also bullying", "docs": ["Central Intelligence", "The Substitute"], "original_query": "Films about bullying that are also both Films about security and surveillance and American action films", "scores": null, "metadata": {"template": "_ that are also both _ and _", "domain": "films", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Central Intelligence": ["Likely relevant"], "The Substitute": ["Definitely relevant"]}, "evidence_ratings": {"Central Intelligence": ["Partial"], "The Substitute": ["Partial"]}, "attributions": {"Central Intelligence": [{"American action films about security and surveillance": "'''''Central Intelligence''''' is a 2016 American buddy action comedy film directed by Rawson Marshall Thurber and written by Thurber, Ike Barinholtz and David Stassen. The film stars Kevin Hart and Dwayne Johnson as two old high school classmates who go on the run after one of them joins the CIA to save the world from a terrorist who intends to sell satellite codes."}], "The Substitute": [{"American action films about security and surveillance also bullying": "Shale sets up surveillance cameras throughout the building."}]}}} +{"query": "Neogene mammals of Africa that are Odd-toed ungulates", "docs": ["Eurygnathohippus", "Victoriaceros", "Brachypotherium", "Paradiceros", "Diceros praecox", "Chemositia", "Hippotherium", "Ancylotherium", "Rusingaceros", "Ceratotherium mauritanicum"], "original_query": "Odd-toed ungulates that are also Neogene mammals of Africa", "scores": null, "metadata": {"template": "_ that are also _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "animals"}} +{"query": "Films set in Pittsburgh that are LGBT-related romance", "docs": ["Anything's Possible (film)", "Happiest Season", "The Perks of Being a Wallflower (film)"], "original_query": "LGBT-related romance films that are also Films set in Pittsburgh", "scores": null, "metadata": {"template": "_ that are also _", "domain": "films", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Anything's Possible (film)": ["Likely relevant"], "Happiest Season": ["Definitely relevant"], "The Perks of Being a Wallflower (film)": ["Likely relevant"]}, "evidence_ratings": {"Anything's Possible (film)": ["Complete"], "Happiest Season": ["Complete"], "The Perks of Being a Wallflower (film)": ["Partial"]}, "attributions": {"Anything's Possible (film)": [{"romance": "'''''Anything's Possible''''' is an upcoming American coming-of-age romantic comedy film", "LGBT-related": "The premise has been compared to the likes of ''Love, Simon'' and ''Booksmart''.", "Films set in Pittsburgh": "e agreed to sign on 30 pages into the script upon realizing the film would be set in his hometown of Pittsburgh."}], "Happiest Season": [{"Films set in Pittsburgh that are LGBT-related romance": "'''''Happiest Season''''' is a 2020 American holiday romantic comedy-drama film directed by Clea DuVall, from a screenplay written by DuVall and Mary Holland. With an ensemble cast consisting of Kristen Stewart, Mackenzie Davis, Alison Brie, Aubrey Plaza, Dan Levy, Mary Holland, Victor Garber, and Mary Steenburgen, the film follows a woman who struggles to admit to her conservative parents that she's a lesbian while she and her girlfriend visit them during Christmas. DuVall has said the film is a semi-autobiographical take on her own experiences with her family.\n\n\nProduced by TriStar Pictures, the film was released in the United States on November 25, 2020, by Hulu, and internationally on November 26, by Sony Pictures Releasing and Entertainment One. The film received positive reviews from critics, with praise for the cast. The film won a GLAAD Media Award in April 2021 in the \"Outstanding Film - Wide Release\" category.\nAbby Holland and Harper Caldwell are a couple who have been dating nearly a year. Abby has disliked Christmas since her parents died, so Harper spontaneously invites Abby to celebrate the holidays with her family in her hometown. Abby sees this as the perfect opportunity to introduce herself to Harper's parents and propose to her on Christmas morning. However, on their way to the Caldwells' house, Harper reveals she lied to Abby about coming out to her parents previously; fearing doing so would interfere with her father's campaign for mayor, she doesn't want to come out to her family until after Christmas. She asks Abby to pretend to be her straight roommate for the holiday, to which Abby reluctantly agrees."}], "The Perks of Being a Wallflower (film)": [{"Films set in Pittsburgh that are LGBT-related romance": "Sam was based on girls who confided in him, and Patrick was \"all the kids I knew who were gay and finding their way to their own identity.\""}]}}} +{"query": "find me Epistemology books", "docs": ["Word and Object", "Aenesidemus (book)"], "original_query": "Epistemology books", "scores": null, "metadata": {"template": "_", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Word and Object": ["Definitely relevant"], "Aenesidemus (book)": ["Likely relevant"]}, "evidence_ratings": {"Word and Object": ["Complete"], "Aenesidemus (book)": ["Partial"]}, "attributions": {"Word and Object": [{"Epistemology books": "He argues in favor of naturalizing epistemology, supports physicalism over phenomenalism and mind-body dualism, and extensionality over intensionality, develops a behavioristic conception of sentence-meaning, theorizes about language learning, speculates on the ontogenesis of reference, explains various forms of ambiguity and vagueness, recommends measures for regimenting language to eliminate ambiguity and vagueness as well as to make perspicuous the logic and ontic commitments of theories, argues against quantified modal logic and the essentialism it presupposes, argues for Platonic realism in mathematics, rejects instrumentalism in favor of scientific realism, develops a view of philosophical analysis as explication, argues against analyticity and for holism, against countenancing propositions, and tries to show that the meanings of theoretical sentences are indeterminate and that the reference of terms is inscrutable."}], "Aenesidemus (book)": [{"Epistemology": "Skepticism doubts the possibility of knowledge about the existence or non-existence of the thing-in-itself."}]}}} +{"query": "Snakes of Asia that are Fauna of Oceania and Reptiles of the Philippines", "docs": ["Yellow-lipped sea krait", "Acrochordus granulatus", "Dendrelaphis caudolineatus", "Blue-lipped sea krait", "Stokes's sea snake", "Hydrophis peronii"], "original_query": "Snakes of Asia that are also both Fauna of Oceania and Reptiles of the Philippines", "scores": null, "metadata": {"template": "_ that are also both _ and _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "animals"}} +{"query": "Brazilian fantasy films,or shot in Amazonas", "docs": ["Good Manners (film)", "Xuxa e os Duendes", "Xuxa em Sonho de Menina", "Oriundi (film)", "Past Minutes", "Where the River Runs Black", "Macuna\u00edma (film)", "Xuxinha e Guto contra os Monstros do Espa\u00e7o", "Burden of Dreams", "Xuxa Abracadabra", "O Saci (film)", "Tain\u00e1 2: A New Amazon Adventure", "Diamantino (film)", "Xuxa e os Duendes 2: No Caminho das Fadas", "Fitzcarraldo", "Xuxa e o Tesouro da Cidade Perdida", "Anaconda (film)"], "original_query": "Films shot in Amazonas (Brazilian state) or Brazilian fantasy films", "scores": null, "metadata": {"template": "_ or _", "domain": "films", "fluency": ["Mostly Fluent: It has a few errors or it does not sound natural, but I can understand it."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Good Manners (film)": ["Definitely relevant"], "Xuxa e os Duendes": ["Definitely relevant"], "Xuxa em Sonho de Menina": ["Definitely relevant"], "Oriundi (film)": ["Likely relevant"], "Past Minutes": ["Definitely relevant"], "Where the River Runs Black": ["Likely relevant"], "Macuna\u00edma (film)": ["Definitely relevant"], "Xuxinha e Guto contra os Monstros do Espa\u00e7o": ["Likely relevant"], "Burden of Dreams": ["Definitely relevant"], "Xuxa Abracadabra": ["Definitely relevant"], "O Saci (film)": ["Definitely relevant"], "Tain\u00e1 2: A New Amazon Adventure": ["Definitely relevant"], "Diamantino (film)": ["Definitely relevant"], "Xuxa e os Duendes 2: No Caminho das Fadas": ["Likely relevant"], "Fitzcarraldo": ["Likely relevant"], "Xuxa e o Tesouro da Cidade Perdida": ["Definitely relevant"], "Anaconda (film)": ["Definitely relevant"]}, "evidence_ratings": {"Good Manners (film)": ["Complete"], "Xuxa e os Duendes": ["Complete"], "Xuxa em Sonho de Menina": ["Partial"], "Oriundi (film)": ["Partial"], "Past Minutes": ["Partial"], "Where the River Runs Black": ["Partial"], "Macuna\u00edma (film)": ["Complete"], "Xuxinha e Guto contra os Monstros do Espa\u00e7o": ["Partial"], "Burden of Dreams": ["Complete"], "Xuxa Abracadabra": ["Complete"], "O Saci (film)": ["Complete"], "Tain\u00e1 2: A New Amazon Adventure": ["Complete"], "Diamantino (film)": ["Complete"], "Xuxa e os Duendes 2: No Caminho das Fadas": ["Partial"], "Fitzcarraldo": ["Complete"], "Xuxa e o Tesouro da Cidade Perdida": ["Complete"], "Anaconda (film)": ["Partial"]}, "attributions": {"Good Manners (film)": [{"Brazilian fantasy films": "'''''Good Manners''''' () is a 2017 dark fantasy horror film written and directed by Juliana Rojas and Marco Dutra"}], "Xuxa e os Duendes": [{"Brazilian fantasy films": "'''Xuxa e os Duendes''' () is a 2001 Brazilian fantasy\nmusical adventure children's film directed by Paulo S\u00e9rgio de Almeida, Rog\u00e9rio Gomes and M\u00e1rcio Vito with a screenplay by Vivian Perl and Wagner de Assis."}], "Xuxa em Sonho de Menina": [{"Brazilian fantasy films": "'''Xuxa em Sonho de Menina''' () is a 2007 Brazilian fantasy children's film"}], "Oriundi (film)": [{"Brazilian fantasy films": "'''''Oriundi''''' is a 2000 Brazilian drama film"}], "Past Minutes": [{"Brazilian fantasy films,or shot in Amazonas": "'''''Past Minutes''''' (Portuguese: '''''Minutos Atr\u00e1s''''') is a 2013 Brazilian comedy-fantasy film"}], "Where the River Runs Black": [{"shot in Amazonas": "'''''Where the River Runs Black''''' is a 1986 film directed by Christopher Cain and starring Charles Durning, Peter Horton, and Dana Delany. The screenplay was written by Neal Jimenez and Peter Silverman, based on the novel ''Lazaro'' by David Kendall.\nBrazil, 1976. Father Mahoney (Peter Horton) is a missionary priest in the Amazon, and on occasion reports to his superior, Father O'Reilly (Charles Durning), who disapproves of his plans to save the indigenous people of the area through medicine and development."}], "Macuna\u00edma (film)": [{"Brazilian fantasy films,or shot in Amazonas": "'''''Macuna\u00edma''''' is a 1969 Brazilian comedy film directed by Joaquim Pedro de Andrade, based on M\u00e1rio de Andrade's novel of the same name. It was released in a dubbed version for American audiences in 1972 by New Line Cinema. On June 13 and July 12, 2005, European and Latin American syndicates of the TV5 network aired the film in its original Portuguese with French subtitles. It was rereleased internationally in 2009.\nBased on the 1928 book by M\u00e1rio de Andrade, the modern-day parable follows the misadventures of a black man (Grande Otelo) who is miraculously born to an old woman (Paulo Jos\u00e9), who is supposed to be of the indigenous peoples of Brasil, in the jungles of the Amazon. Though born fully-grown, he has the heart of a playful child. After the death of his mother, he comes face to face with a spring that turns him white (Paulo Jos\u00e9). With that change, he and his two brothers move to Rio de Janeiro, but are interrogated by street terrorists upon their arrival."}], "Xuxinha e Guto contra os Monstros do Espa\u00e7o": [{"Brazilian fantasy films": "'''''Xuxinha e Guto contra os Monstros do Espa\u00e7o''''' () is a 2005 Brazilian computer-animated adventure film written by Fl\u00e1vio de Souza and directed by Moacyr G\u00f3es and Clewerson Saremba, produced by Diler Trindade and distributed by Warner Bros. Pictures. Starring Xuxa Meneghel and Pedro Malta with the participation of the voice actors Fl\u00e1via Saddy, Bernardo Coutinho, Guilherme Briggs, Gustavo Pereira, Milton Gon\u00e7alves."}], "Burden of Dreams": [{"shot in Amazonas": "a 1982 \"making-of\" documentary film directed by Les Blank, shot during and about the chaotic production of Werner Herzog's 1982 film ''Fitzcarraldo'', and filmed on location in the jungles of Peru."}], "Xuxa Abracadabra": [{"Brazilian fantasy films": "'''''Xuxa Abracadabra''''' is a 2003 Brazilian fantasy adventure children's filmis, written by Fl\u00e1vio de Souza, directed by Moacyr G\u00f3es, produced by Diler Trindade and distributed by Warner Bros."}], "O Saci (film)": [{"Brazilian fantasy films": "a 1951 Brazilian children-family fantasy film"}], "Tain\u00e1 2: A New Amazon Adventure": [{"shot in Amazonas": "It was shot in Manaus, the capital of the state of Amazonas,"}], "Diamantino (film)": [{"fantasy films": "'''''Diamantino''''' is a 2018 fantasy comedy-drama film written and directed by Gabriel Abrantes and Daniel Schmidt.", "Brazilian": "The film was shot in Portugal, France, and Brazil through their respective production companies."}], "Xuxa e os Duendes 2: No Caminho das Fadas": [{"Brazilian fantasy films,": "'''''Xuxa e os Duendes 2: No Caminho das Fadas''''' () is a 2002 Brazilian fantasy adventure children's film directed by Paulo S\u00e9rgio de Almeida, Rog\u00e9rio Gomes and M\u00e1rcio Vito and written by Vivian Perl and Wagner de Assis. It is the sequel to ''Xuxa e os Duendes'' (2001). Xuxa Meneghel, Ana Maria Braga, Debby Lagranha, Emiliano Queiroz, Tadeu Mello and Guilherme Karan reprise their respective roles from the first film, joined by Luciano Szafir, Betty Lago, Vera Fischer, Deborah Secco, Thiago Fragoso. In this sequel, Kira (Xuxa), the elf of light, is in charge of saving her friends and mankind from a curse that would petrify the hearts of lovers and end love on the face of the Earth."}], "Fitzcarraldo": [{"shot in Amazonas": "It portrays would-be rubber baron Brian Sweeney Fitzgerald, an Irishman known in Peru as Fitzcarraldo, who is determined to transport a steamship over a steep hill to access a rich rubber territory in the Amazon Basin."}], "Xuxa e o Tesouro da Cidade Perdida": [{"Brazilian fantasy films": "'''''Xuxa e o Tesouro da Cidade Perdida''''' () is a 2004 Brazilian fantasy adventure children's film, written by Fl\u00e1vio de Souza, directed by Moacyr G\u00f3es, produced by Diler Trindade and distributed by Warner Bros."}], "Anaconda (film)": [{"or shot in Amazonas": "It focuses on a documentary film crew in the Amazon rainforest"}]}}} +{"query": "1997 anime films or slayer films", "docs": ["Neon Genesis Evangelion: Death & Rebirth", "The End of Evangelion", "Case Closed: The Time Bombed Skyscraper", "Princess Mononoke", "Crayon Shin-chan: Pursuit of the Balls of Darkness", "Jungle Emperor Leo", "Slayers Great", "My Father's Dragon (1997 film)", "Slayers Premium", "Perfect Blue", "Slayers Gorgeous", "Tenchi the Movie 2: The Daughter of Darkness", "Slayers The Motion Picture", "Slayers Return"], "original_query": "Slayers films or 1997 anime films", "scores": null, "metadata": {"template": "_ or _", "domain": "films", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Neon Genesis Evangelion: Death & Rebirth": ["Definitely relevant"], "The End of Evangelion": ["Likely relevant"], "Case Closed: The Time Bombed Skyscraper": ["Likely relevant"], "Princess Mononoke": ["Definitely relevant"], "Crayon Shin-chan: Pursuit of the Balls of Darkness": ["Definitely relevant"], "Jungle Emperor Leo": ["Definitely relevant"], "Slayers Great": ["Definitely relevant"], "My Father's Dragon (1997 film)": ["Likely relevant"], "Slayers Premium": ["Definitely relevant"], "Perfect Blue": ["Definitely relevant"], "Slayers Gorgeous": ["Likely relevant"], "Tenchi the Movie 2: The Daughter of Darkness": ["Definitely relevant"], "Slayers The Motion Picture": ["Definitely relevant"], "Slayers Return": ["Definitely relevant"]}, "evidence_ratings": {"Neon Genesis Evangelion: Death & Rebirth": ["Complete"], "The End of Evangelion": ["No evidence"], "Case Closed: The Time Bombed Skyscraper": ["Partial"], "Princess Mononoke": ["Partial"], "Crayon Shin-chan: Pursuit of the Balls of Darkness": ["Complete"], "Jungle Emperor Leo": ["Complete"], "Slayers Great": ["Complete"], "My Father's Dragon (1997 film)": ["Partial"], "Slayers Premium": ["Complete"], "Perfect Blue": ["Complete"], "Slayers Gorgeous": ["Complete"], "Tenchi the Movie 2: The Daughter of Darkness": ["Complete"], "Slayers The Motion Picture": ["Complete"], "Slayers Return": ["Complete"]}, "attributions": {"Neon Genesis Evangelion: Death & Rebirth": [{"1997 anime films": ", also romanized in Japan as ''Evangelion:Death'' and ''Evangelion:Rebirth'', is a 1997 animated science fiction psychological drama film."}], "The End of Evangelion": [null], "Case Closed: The Time Bombed Skyscraper": [{"1997 anime films": "'''''Case Closed: The Time Bombed Skyscraper''''', known as in Japan, is a 1997 Japanese animated feature film based on Gosho Aoyama's ''Case Closed'' manga series and featuring detective Jimmy Kudo. The film takes place between episodes 54 and 55. Funimation released the film in North America."}], "Princess Mononoke": [{"1997 anime films": "is a 1997 Japanese animated epic historical fantasy film"}], "Crayon Shin-chan: Pursuit of the Balls of Darkness": [{"1997 anime films or slayer films": "also known as '''Dark Tamatam Thrilling Chase!''', is a 1997 Japanese animated film and the fifth installment of the ''Crayon Shin-chan'' series"}], "Jungle Emperor Leo": [{"1997 anime films": "'''''Jungle Emperor Leo''''', known in Japan as is a 1997 Japanese animated film focusing on the last half of Osamu Tezuka's manga, ''Jungle Taitei'' (known in earlier US productions as ''Kimba the White Lion'' and ''Leo the Lion'')."}], "Slayers Great": [{"1997 anime films": "is a 1997 anime film written by Hajime Kanzaka and directed by Kunihiko Yuyama and Hiroshi Watanabe."}], "My Father's Dragon (1997 film)": [{"1997 anime films": "is a 1997 Japanese animated film based on the ''My Father's Dragon'' children's books by Ruth Stiles Gannett and their illustrations by Ruth Chrisman Gannett."}], "Slayers Premium": [{"slayer films": "is a 2001 Japanese anime short film written and directed by Junichi Sat\u014d. ''Premium'', the fifth and latest ''Slayers'' film, was first released in Japan on December 22, 2001, along with ''Sakura Wars: The Movie'', ''Di Gi Charat - A Trip to the Planet'' and ''Azumanga Daioh: The Animation''."}], "Perfect Blue": [{"1997 anime films": "is a 1997 Japanese animated psychological thriller film directed by Satoshi Kon."}], "Slayers Gorgeous": [{"slayer films": "a 1998 comic fantasy anime film written by Hajime Kanzaka and directed by Hiroshi Watanabe. ''Gorgeous'' is the fourth film in the ''Slayers'' saga and the final anime entry in the prequels to the main series."}], "Tenchi the Movie 2: The Daughter of Darkness": [{"1997 anime films": "The film was released in Japan on August 2, 1997 as a double feature, along with ''Slayers Great''.", "anime": "Funimation announced distribution of the film, along with several other ''Tenchi'' properties, on July 2, 2010 at Anime Expo."}], "Slayers The Motion Picture": [{"slayer films": "'''''Slayers \u2013 The Motion Picture''''', also known as '''''Slayers Perfect''''' (or '''''Slayers the Movie: Perfect Edition''''') and originally released in Japan simply as , is a 1995 Japanese animated comic fantasy adventure film directed by Hiroshi Watanabe and written by Kazuo Yamazaki, based on the novels by Hajime Kanzaka."}], "Slayers Return": [{"slayer films": ", also known as ''Slayers Movie 2 - The Return'', is a 1996 Japanese comic fantasy anime film written by Hajime Kanzaka and directed by Kunihiko Yuyama and Hiroshi Watanabe. It is the second film released in the ''Slayers'' saga and a sequel to 1995's ''Slayers Perfect'' that was itself followed by ''Slayers Great'' in 1997."}]}}} +{"query": "what are Australian travel books", "docs": ["Through Darkest Pondelayo", "San Sombr\u00e8ro", "The Great Trek (book)", "30 Days in Sydney", "Molvan\u00eea", "Local Colour: Travels in the Other Australia", "Phaic T\u0103n", "Down Under (book)"], "original_query": "Australian travel books", "scores": null, "metadata": {"template": "_", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Through Darkest Pondelayo": ["Definitely relevant"], "San Sombr\u00e8ro": ["Definitely relevant"], "The Great Trek (book)": ["Definitely relevant"], "30 Days in Sydney": ["Definitely relevant"], "Molvan\u00eea": ["Definitely relevant"], "Local Colour: Travels in the Other Australia": ["Definitely relevant"], "Phaic T\u0103n": ["Definitely relevant"], "Down Under (book)": ["Definitely relevant"]}, "evidence_ratings": {"Through Darkest Pondelayo": ["Complete"], "San Sombr\u00e8ro": ["Complete"], "The Great Trek (book)": ["Complete"], "30 Days in Sydney": ["Complete"], "Molvan\u00eea": ["Partial"], "Local Colour: Travels in the Other Australia": ["Complete"], "Phaic T\u0103n": ["Partial"], "Down Under (book)": ["Complete"]}, "attributions": {"Through Darkest Pondelayo": [{"Australian": "'''''Through Darkest Pondelayo: An account of the adventures of two English ladies on a cannibal island ''''' is a 1936 Australian satirical novel by Joan Lindsay, published under the pseudonym Serena Livingstone-Stanley.", "travel books": ". It is structured in the format of a travel book through a series of first-person letters edited together to form a metafictional narrative."}], "San Sombr\u00e8ro": [{"Australian": "This country is set in Central America, and was created by Australian comedic writers Tom Gleisner, Santo Cilauro and Rob Sitch (of ''The D-Generation'' and ''The Panel'' fame).", "travel books": "'''''San Sombr\u00e8ro''''' (subtitled ''A Land of Carnivals, Cocktails and Coups'') is a parody travel guide book examining the eponymous fictional country, described as the birthplace of tinted sunglasses and sequins."}], "The Great Trek (book)": [{"what are Australian travel books": "'''''The Great Trek: One of the Greatest Feats in Australian Exploration"}], "30 Days in Sydney": [{"what are Australian travel books": "'''''30 Days in Sydney''''' is a book written by Australian novelist Peter Carey. It was published in 2001 and is subtitled ''A Wildly Distorted Account''."}], "Molvan\u00eea": [{"what are Australian travel books": "'''Molvan\u00eea''''' (subtitled ''A Land Untouched by Modern Dentistry'') is a book parodying travel guidebooks. The guide describes the fictional country Molvan\u00eea, a post-Soviet state, a nation described as \"the birthplace of the whooping cough\" and \"owner of Europe's oldest nuclear reactor\". It was created by Australians Tom Gleisner, Santo Cilauro and Rob Sitch (locally known for ''The D-Generation'' and ''The Panel'' in Australia)."}], "Local Colour: Travels in the Other Australia": [{"Australian travel books": "'''''Local Colour: Travels in the Other Australia''''' is a book containing photography and text by Bill Bachman with additional text by Tim Winton."}], "Phaic T\u0103n": [{"what are Australian travel books": "'''''Phaic T\u0103n''''' (subtitled ''Sunstroke on a Shoestring'') is a 2004 parody travel guidebook examining imaginary country Phaic T\u0103n. The book was written by Australians Tom Gleisner, Santo Cilauro, and Rob Sitch. Along with the other Jetlag Travel volumes, 2003's ''Molvan\u00eea'' and 2006's ''San Sombr\u00e8ro'', the book parodies both the language of heritage tourism and the legacy of colonialism and imperialism."}], "Down Under (book)": [{"Australian travel books": "'''''Down Under''''' is the British title of a 2000 travelogue book about Australia written by best-selling travel writer Bill Bryson."}]}}} +{"query": "Set in the Edo period but not Japanese historical films", "docs": ["The King Kong That Appeared in Edo", "Kamen Rider OOO Wonderful: The Shogun and the 21 Core Medals", "Humanity and Paper Balloons", "Absolutely Secret: Girl Torture"], "original_query": "Films set in the Edo period that are not Japanese historical films", "scores": null, "metadata": {"template": "_ that are not _", "domain": "films", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"The King Kong That Appeared in Edo": ["Definitely relevant"], "Kamen Rider OOO Wonderful: The Shogun and the 21 Core Medals": ["Likely relevant"], "Humanity and Paper Balloons": ["Definitely relevant"], "Absolutely Secret: Girl Torture": ["Definitely relevant"]}, "evidence_ratings": {"The King Kong That Appeared in Edo": ["Complete"], "Kamen Rider OOO Wonderful: The Shogun and the 21 Core Medals": ["No evidence"], "Humanity and Paper Balloons": ["Complete"], "Absolutely Secret: Girl Torture": ["Complete"]}, "attributions": {"The King Kong That Appeared in Edo": [{"Set in the Edo period": "is a 1938 Japanese two-part silent period piece drama film set during the Edo period and produced by Zensh\u014d Cinema."}], "Kamen Rider OOO Wonderful: The Shogun and the 21 Core Medals": [null], "Humanity and Paper Balloons": [{"Set in the Edo period but not Japanese historical films": "The film is set in feudal Japan during the 18th century, an era known as the Edo period.", "not Japanese historical films": "It depicts the struggles and schemes of Matajuro Unno, a r\u014dnin, or masterless samurai, and his neighbor Shinza, a hairdresser."}], "Absolutely Secret: Girl Torture": [{"Set in the Edo period but not Japanese historical films": "''aka'' '''''Top Secrets of Women Torture''''' and '''''Top Secret of Torturing Women''''' is a 1968 Japanese ''Pink film'' in the ''ero guro'' style directed by Kiyoshi Komori ''aka'' Haku Komori. The film features future Nikkatsu SM-queen Naomi Tani in a role during the first half of her career, working outside of the large studio system.\nSet in the Edo Era, the film opens with a group of women being convicted of various crimes. The rest of the film is given to graphic depiction of the tortures the women endure as part of their sentences."}]}}} +{"query": "Books about Brunei, Malaysia or historical books about the Qing dynasty", "docs": ["Intellectual Trends in the Qing Period", "Malaysia and the Club of Doom", "Kalimantaan", "Treason by the Book", "May 13: Declassified Documents on the Malaysian Riots of 1969", "Empress Dowager Cixi: The Concubine Who Launched Modern China", "Devil of a State", "Chronicle of Malaysia", "The Malay Dilemma"], "original_query": "Books about Brunei or Books about Malaysia or History books about the Qing dynasty", "scores": null, "metadata": {"template": "_ or _ or _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Intellectual Trends in the Qing Period": ["Definitely relevant"], "Malaysia and the Club of Doom": ["Definitely relevant"], "Kalimantaan": ["Definitely relevant"], "Treason by the Book": ["Definitely relevant"], "May 13: Declassified Documents on the Malaysian Riots of 1969": ["Definitely relevant"], "Empress Dowager Cixi: The Concubine Who Launched Modern China": ["Definitely relevant"], "Devil of a State": ["Definitely relevant"], "Chronicle of Malaysia": ["Definitely relevant"], "The Malay Dilemma": ["Likely relevant"]}, "evidence_ratings": {"Intellectual Trends in the Qing Period": ["Complete"], "Malaysia and the Club of Doom": ["Partial"], "Kalimantaan": ["Partial"], "Treason by the Book": ["Complete"], "May 13: Declassified Documents on the Malaysian Riots of 1969": ["Partial"], "Empress Dowager Cixi: The Concubine Who Launched Modern China": ["Partial"], "Devil of a State": ["Complete"], "Chronicle of Malaysia": ["Complete"], "The Malay Dilemma": ["Partial"]}, "attributions": {"Intellectual Trends in the Qing Period": [{"historical books about the Qing dynasty": "'''''Intellectual Trends in the Qing Period''''' or ''An Overview of Scholarship in Qing'' (), also translated as ''An Outline of Academic Studies in the Qing Dynasty'', ''General Review of Academics in Qing Dynasty'', is a book on the history of scholarship in the Qing dynasty by Liang Qichao that was started in 1902 and published in 1920."}], "Malaysia and the Club of Doom": [{"Malaysia": "With a particular focus on Malaysia, the thesis of the book is that growing Islamisation of the political and social climate"}], "Kalimantaan": [{"Brunei,": "an English adventurer arrived on the northwest coast of Borneo, commissioned to deliver a letter of gratitude to the Sultan of Brunei"}], "Treason by the Book": [{"Qing dynasty": "'''''Treason by the Book''''', by Jonathan Spence, is a historical account of the Zeng Jing (\u66fe\u975c) case which took place during the reign of the Yongzheng Emperor of Qing China around 1730."}], "May 13: Declassified Documents on the Malaysian Riots of 1969": [{"Books about Brunei, Malaysia": "'''''May 13: Declassified Documents on the Malaysian Riots of 1969''''' is a book published in 2007 and written by activist and scholar Dr. Kua Kia Soong on the 13 May Incident of 1969. It was published by the human rights group Suaram on the 38th anniversary of the worst racial riot in Malaysian history, which took place mostly in Kuala Lumpur. The official death toll was 196, but independent reporters and other observers estimated up to ten times as many people had died. Three quarters of the casualties were Chinese Malaysians, and 6000 of them were left homeless after fires"}], "Empress Dowager Cixi: The Concubine Who Launched Modern China": [{"historical books about the Qing dynasty": "Chang presents a sympathetic portrait of the Empress Dowager Cixi, who unofficially controlled the Manchu Qing Dynasty in China for 47 years, from 1861 to her death in 1908."}], "Devil of a State": [{"Books about Brunei, Malaysia": "'''''Devil of a State''''' is a 1961 novel by Anthony Burgess based on his experience living and working in Bandar Seri Begawan, the capital of the Southeast Asian sultanate of Brunei, on the island of Borneo, in 1958-59."}], "Chronicle of Malaysia": [{"Malaysia": "The '''''Chronicle of Malaysia''''' gives an account of Malaysia from January 1957 to 31 August 2007."}], "The Malay Dilemma": [{"Books about Brunei, Malaysia or historical books about the Qing dynasty": "In it, Mahathir describes his interpretation of Malaysia's history in relation to its multi-ethnic society and racial tensions, and advocates for affirmative action policies for Malays."}]}}} diff --git a/baseline_quest/data/train_subset2.jsonl b/baseline_quest/data/train_subset2.jsonl new file mode 100644 index 0000000..ca2b289 --- /dev/null +++ b/baseline_quest/data/train_subset2.jsonl @@ -0,0 +1,20 @@ +{"query": "Garden plants that are from both Asia, Australasia and Malesia.", "docs": ["Asplenium nidus", "Cordyline fruticosa"], "original_query": "Garden plants of Asia that are also both Garden plants of Australasia and Flora of Malesia", "scores": null, "metadata": {"template": "_ that are also both _ and _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "plants"}} +{"query": "1992 American children's comedy animated musical films", "docs": ["FernGully: The Last Rainforest", "Aladdin (1992 Disney film)", "Tom and Jerry: The Movie"], "original_query": "1992 films that are also both Animated musical films and American children's comedy films", "scores": null, "metadata": {"template": "_ that are also both _ and _", "domain": "films", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"FernGully: The Last Rainforest": ["Likely relevant"], "Aladdin (1992 Disney film)": ["Definitely relevant"], "Tom and Jerry: The Movie": ["Definitely relevant"]}, "evidence_ratings": {"FernGully: The Last Rainforest": ["Partial"], "Aladdin (1992 Disney film)": ["Complete"], "Tom and Jerry: The Movie": ["Complete"]}, "attributions": {"FernGully: The Last Rainforest": [{"1992": "'''''FernGully: The Last Rainforest''''' is a 1992 animated musical fantasy film, directed by Bill Kroyer and scripted by Jim Cox.", "children's": "'''''FernGully: The Last Rainforest''''' is a 1992 animated musical fantasy film, directed by Bill Kroyer and scripted by Jim Cox", "animated musical films": "'''''FernGully: The Last Rainforest''''' is a 1992 animated musical fantasy film, directed by Bill Kroyer and scripted by Jim Cox.", "American": "Adapted from the book of the same name by Diana Young, the film is an Australian and American venture produced by Kroyer Films, Inc., Youngheart Productions, FAI Films and 20th Century Fox."}], "Aladdin (1992 Disney film)": [{"1992 American children's comedy animated musical films": "'''''Aladdin''''' is a 1992 American animated musical fantasy comedy film produced by Walt Disney Feature Animation and released by Walt Disney Pictures. The film is the 31st Disney animated feature film and was the fourth produced during the Disney Renaissance. It was produced and directed by Ron Clements and John Musker, and is based on the Arabic folktale of the same name from the ''One Thousand and One Nights''. The voice cast features Scott Weinger, Robin Williams, Linda Larkin, Jonathan Freeman, Frank Welker, Gilbert Gottfried, and Douglas Seale. The film follows the titular Aladdin, an Arabian street urchin, who finds a magic lamp containing a genie. He disguises himself as a wealthy prince and tries to impress the Sultan in order to marry his free-spirited daughter, Princess Jasmine."}], "Tom and Jerry: The Movie": [{"1992 American children's comedy animated musical films": "'''''Tom and Jerry: The Movie''''' is a 1992 American animated musical comedy film based on the characters ''Tom and Jerry'' created by William Hanna and Joseph Barbera."}]}}} +{"query": "Spy novels about North America and the military", "docs": ["The Death Strain", "A Matter of Time (Cook novel)", "The Man from Barbarossa", "The Cardinal of the Kremlin", "The Sympathizer", "Camp X (novel)", "The Man Next Door (novel)", "Mongoose R.I.P.", "The Betrayers", "The Man with the Golden Gun (novel)", "Kolymsky Heights", "A Tract of Time"], "original_query": "Books about North America that are also both Novels about the military and Spy novels", "scores": null, "metadata": {"template": "_ that are also both _ and _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"The Death Strain": ["Definitely relevant"], "A Matter of Time (Cook novel)": ["Definitely relevant"], "The Man from Barbarossa": ["Likely relevant"], "The Cardinal of the Kremlin": ["Definitely relevant"], "The Sympathizer": ["Definitely relevant"], "Camp X (novel)": ["Definitely relevant"], "The Man Next Door (novel)": ["Definitely relevant"], "Mongoose R.I.P.": ["Definitely relevant"], "The Betrayers": ["Likely relevant"], "The Man with the Golden Gun (novel)": ["Definitely relevant"], "Kolymsky Heights": ["Definitely relevant"], "A Tract of Time": ["Definitely relevant"]}, "evidence_ratings": {"The Death Strain": ["Complete"], "A Matter of Time (Cook novel)": ["Complete"], "The Man from Barbarossa": ["Partial"], "The Cardinal of the Kremlin": ["Complete"], "The Sympathizer": ["Complete"], "Camp X (novel)": ["Complete"], "The Man Next Door (novel)": ["Partial"], "Mongoose R.I.P.": ["Complete"], "The Betrayers": ["Partial"], "The Man with the Golden Gun (novel)": ["Complete"], "Kolymsky Heights": ["Complete"], "A Tract of Time": ["Complete"]}, "attributions": {"The Death Strain": [{"Spy novels about North America and the military": "Carter is a US secret agent, code-named N-3, with the rank of Killmaster."}], "A Matter of Time (Cook novel)": [{"Spy novels about North America and the military": "'''''A Matter of Time''''' is a novel by Glen Cook, combining elements of science fiction (specifically, time travel), crime fiction and spy thriller. In regard to the last, the novel in particular takes up and expands the theme of American prisoners of war being brainwashed in Communist China and their loyalties reversed \u2013 a theme made famous through the novel ''The Manchurian Candidate'' and film made on its basis."}], "The Man from Barbarossa": [{"Spy novels about North America and the military": "'''''The Man from Barbarossa''''', first published in 1991, was the eleventh novel by John Gardner featuring Ian Fleming's secret agent, James Bond"}], "The Cardinal of the Kremlin": [{"Spy novels": "'''''The Cardinal of the Kremlin''''' is an espionage thriller novel, written by Tom Clancy and released on May 20, 1988.", "the military": "A direct sequel to ''The Hunt for Red October'' (1984), it features CIA analyst Jack Ryan as he extracts CARDINAL, the agency's highest placed agent in the Soviet government who is being pursued by the KGB, as well as the Soviet intelligence agency's director."}], "The Sympathizer": [{"Spy novels about North America and the military": "'''''The Sympathizer''''' is the 2015 debut novel by Vietnamese-American professor Viet Thanh Nguyen. It is a best-selling novel and recipient of the 2016 Pulitzer Prize for Fiction. The novel received generally positive acclaim from critics, and it was named a ''New York Times'' Editor's Choice.\n\n\nThe novel incorporates elements from a number of different novel genres: immigrant, mystery, political, metafiction, dark comedic, historical, spy, and war."}], "Camp X (novel)": [{"Spy novels about North America and the military": "'''''Camp X''''' is a children's spy novel written by Canadian author Eric Walters. Set in World War II, the novel is about the two brothers Jack and George, trying to save a top-secret Canadian military base called Camp X. They accidentally discovered it after playing a fake game of war."}], "The Man Next Door (novel)": [{"Spy novels about North America and the military": "Smith is obviously an enemy spy and he has a hold on Maida to find out more and more intelligence on wartime materiel and personnel movements."}], "Mongoose R.I.P.": [{"Spy novels about North America and the military": "'''''Mongoose R.I.P.''''' is a 1988 Blackford Oakes novel by William F. Buckley, Jr. It is the eighth of 11 novels in the series.\nCIA agent Blackford Oakes is sent to Cuba to determine the feasibility of overthrowing Fidel Castro, following the Cuban Missile Crisis in 1963."}], "The Betrayers": [{"Spy novels about North America and the military": "'''''The Betrayers''''' is the tenth novel in the Matt Helm spy series by Donald Hamilton, which originated with ''Death of a Citizen'' in 1960. This novel was first published in 1966. It was reissued in 2014 by Titan Books."}], "The Man with the Golden Gun (novel)": [{"Spy novels about North America and the military": "'''''The Man with the Golden Gun''''' is the twelfth and final novel in Ian Fleming's James Bond series and thirteenth Bond book overall. It was first published by Jonathan Cape in the UK on 1 April 1965, eight months after the author's death. The novel was not as detailed or polished as the others in the series, leading to poor but polite reviews. Despite that, the book was a best-seller."}], "Kolymsky Heights": [{"Spy novels about North America and the military": "'''''Kolymsky Heights''''' is a 1994 thriller novel by Lionel Davidson. It was his first thriller novel in 16 years, following ''The Chelsea Murders''.\n\n\nA coded message is smuggled out of Russia, a plea for help from the director of a super-secret laboratory deep in the frozen wastes of Siberia. The note is addressed to Johnny Porter, a Canadian Indian of the Gitxsan tribe with a genius for languages. The CIA recruits Porter, who infiltrates Russia, first posing as a Korean sailor on a tramp freighter, then as a Chukchee driver called Khodyan. Working at a transport company, he befriends an employee who gives him sufficient spare parts to build a bobik truck, which he assembles in a cave. Porter also befriends the local doctor, Tanya Komarova, who is also working for the CIA, and they become lovers. With her help, he infiltrates the research facility by switching places with an Evenk employee. The director, Ephraim Rogachev, reveals to Porter the research they have been conducting, including a cure for blindness which the Soviets are concealing because of the military applications of the technology."}], "A Tract of Time": [{"Spy novels about North America and the military": "'''''A Tract of Time''''' is an antiwar novel from 1966 by Smith Hempstone, that covers the time period about 1960, when there was an attempted coup of South Vietnamese President Ngo Dinh Diem. Even as the United States backed Diem's government during the war, its American advisers worked with the Montagnard people who opposed Diem, to help them fight the Viet Cong, whom they also opposed. The book follows one CIA operative, Harry Coltart, as he works with the Montagnard mountain tribesmen in the Central Highlands. Harry is initially successful in getting the Montagnards to fight against the Viet Cong, but then the Montagnards are betrayed and South Vietnamese troops are sent in. Harry has to be rescued as the Montagnards join the Viet Cong."}]}}} +{"query": "Prehistoric animals of Europe and Africa, that are not from the Miocene period", "docs": ["Columbian mammoth", "Theropithecus oswaldi", "Paracamelus", "Mammuthus meridionalis", "Mammuthus lamarmorai", "Metailurus", "Dinofelis", "Homotherium", "Agriotherium", "Pygmy mammoth", "Steppe mammoth", "Homo erectus", "Pachycrocuta", "Mammuthus rumanus", "Mammuthus creticus", "Necromantis", "Mammuthus subplanifrons"], "original_query": "Mammals of Europe that are also Prehistoric animals of Africa but not Miocene mammals", "scores": null, "metadata": {"template": "_ that are also _ but not _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "animals"}} +{"query": "Children's books published by Grosset & Dunlap in the 1960s", "docs": ["The Password to Larkspur Lane", "The Phantom of Pine Hill", "The Invisible Intruder", "The Clue of the Whistling Bagpipes", "The Bungalow Mystery", "The Clue of the Broken Locket"], "original_query": "1960s children's books that are also Grosset & Dunlap books but not The Hardy Boys books", "scores": null, "metadata": {"template": "_ that are also _ but not _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"The Password to Larkspur Lane": ["Definitely relevant"], "The Phantom of Pine Hill": ["Likely relevant"], "The Invisible Intruder": ["Definitely relevant"], "The Clue of the Whistling Bagpipes": ["Likely relevant"], "The Bungalow Mystery": ["Likely relevant"], "The Clue of the Broken Locket": ["Likely relevant"]}, "evidence_ratings": {"The Password to Larkspur Lane": ["Complete"], "The Phantom of Pine Hill": ["Partial"], "The Invisible Intruder": ["Partial"], "The Clue of the Whistling Bagpipes": ["Partial"], "The Bungalow Mystery": ["Partial"], "The Clue of the Broken Locket": ["Partial"]}, "attributions": {"The Password to Larkspur Lane": [{"Children's books published by Grosset & Dunlap": "Collectors speculate publisher Grosset & Dunlap commissioned an updated illustration of the same scene during the transition from Gillies to new series artist Rudy Nappi in 1953.", "in the 1960s": "This art later appeared on British dust jackets for this volume in 1960."}], "The Phantom of Pine Hill": [{"Children's books published by Grosset & Dunlap in the 1960s": "It was first published in 1965 under the pseudonym Carolyn Keene."}], "The Invisible Intruder": [{"Children's books published by Grosset & Dunlap in the 1960s": "'''''The Invisible Intruder''''' is the 46th volume in the Nancy Drew Mystery Stories series. It was first published in 1969 under Carolyn Keene. The actual author was ghostwriter Harriet Stratemeyer Adams.\nNancy and her friends are invited on a ghost-hunting tour, visiting various locations reputed to be haunted. They gather clues that point to a more mundane explanation."}], "The Clue of the Whistling Bagpipes": [{"Children's books published by Grosset & Dunlap in the 1960s": "It was first published in 1964 under the pseudonym Carolyn Keene."}], "The Bungalow Mystery": [{"in the 1960s": "It was the final volume edited by Edward Stratemeyer before his death. His daughter, Harriet Stratemeyer Adams, extensively revised the novel in 1960."}], "The Clue of the Broken Locket": [{"1960s": "It was later revised by Harriet Stratemeyer in 1965, and the story was mostly changed with a few elements of the original.", "Children's": "he Blairs, who are famous actors, decide to adopt the babies as a publicity stunt, as they hope to raise the children as actors, which will help their own careers."}]}}} +{"query": "World War II prisoner of war films that won a European Film Award", "docs": ["Life Is Beautiful", "Ida (film)"], "original_query": "European Film Awards winners (films) that are also World War II prisoner of war films", "scores": null, "metadata": {"template": "_ that are also _", "domain": "films", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Life Is Beautiful": ["Definitely relevant"], "Ida (film)": ["Definitely relevant"]}, "evidence_ratings": {"Life Is Beautiful": ["Complete"], "Ida (film)": ["Complete"]}, "attributions": {"Life Is Beautiful": [{"World War II prisoner of war films": "The film was partially inspired by the book ''In the End, I Beat Hitler'' by Rubino Romeo Salmon\u00ec and by Benigni's father, who spent two years in the Bergen-Belsen concentration camp during World War II.", "that won a European Film Award": "The movie won the Grand Prix at the 1998 Cannes Film Festival, nine David di Donatello Awards (including Best Film), five Nastro d'Argento Awards in Italy, two European Film Awards, and three Academy Awards, including Best Foreign Language Film and Best Actor for Benigni, the first for a male non-English performance."}], "Ida (film)": [{"World War II prisoner of war films": "Orphaned as an infant during the German occupation of World War II, she must meet her aunt, a former Communist state prosecutor and only surviving relative, who tells her that her parents were Jewish.", "that won a European Film Award": "It had earlier been selected as Best Film of 2014 by the European Film Academy"}]}}} +{"query": "what are Oceanian realm fauna that are also both Birds of North America and Fauna of Europe", "docs": ["Sooty tern", "Bulwer's petrel", "Black noddy", "Bar-tailed godwit", "Masked booby", "Red-footed booby", "Roseate tern"], "original_query": "Oceanian realm fauna that are also both Birds of North America and Fauna of Europe", "scores": null, "metadata": {"template": "_ that are also both _ and _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "animals"}} +{"query": "1970's harpercollins childrens books published posthumously.", "docs": ["The First Four Years (novel)", "The Father Christmas Letters"], "original_query": "1970s children's books that are also both Books published posthumously and HarperCollins books", "scores": null, "metadata": {"template": "_ that are also both _ and _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"The First Four Years (novel)": ["Definitely relevant"], "The Father Christmas Letters": ["Definitely relevant"]}, "evidence_ratings": {"The First Four Years (novel)": ["Complete"], "The Father Christmas Letters": ["Complete"]}, "attributions": {"The First Four Years (novel)": [{"1970's harpercollins childrens books": "The series had initially concluded at eight children's novels following Wilder to mature age and her marriage with Almanzo Wilder."}], "The Father Christmas Letters": [{"1970's harpercollins childrens books published posthumously.": "'''''The Father Christmas Letters''''', also known as '''''Letters from Father Christmas''''', are a collection of letters written and illustrated by J. R. R. Tolkien between 1920 and 1943 for his children, from Father Christmas. They were released posthumously by the Tolkien estate on 2 September 1976, the 3rd anniversary of Tolkien's death. They were edited by Baillie Tolkien, second wife of his youngest son, Christopher. The book was warmly received by critics, and it has been suggested that elements of the stories inspired parts of Tolkien's ''The Lord of the Rings''."}]}}} +{"query": "Cenzoic birds that are not found in New Zealand", "docs": ["Cuban kestrel", "Kaua\u02bbi palila", "Maui Nui finch", "Apteribis", "Scissor-billed koa finch", "Puerto Rican quail-dove", "Aepyornis", "Stilt-owl", "Xestospiza conica", "N\u0113n\u0113-nui", "High-billed crow", "Cuban pauraque", "Xestospiza", "Mullerornis", "Thambetochen", "Xestospiza fastigialis", "Mauritius starling", "Moa-nalo", "Vorombe", "Highland finch"], "original_query": "Endemic fauna that are also Cenozoic birds but not Fauna of New Zealand", "scores": null, "metadata": {"template": "_ that are also _ but not _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "animals"}} +{"query": "1947 Science Linguistics books", "docs": ["LTI \u2013 Lingua Tertii Imperii", "Meaning and Necessity"], "original_query": "Science books that are also both Linguistics books and 1947 books", "scores": null, "metadata": {"template": "_ that are also both _ and _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"LTI \u2013 Lingua Tertii Imperii": ["Likely relevant"], "Meaning and Necessity": ["Definitely relevant"]}, "evidence_ratings": {"LTI \u2013 Lingua Tertii Imperii": ["Partial"], "Meaning and Necessity": ["Complete"]}, "attributions": {"LTI \u2013 Lingua Tertii Imperii": [{"1947 Science Linguistics books": "'''''LTI \u2013 Lingua Tertii Imperii: Notizbuch eines Philologen''''' (1947) is a book by Victor Klemperer, Professor of Literature at the Dresden University of Technology. The title, half in Latin and half in German, translates to \"'''''The Language of the Third Reich: A Philologist's Notebook'''''\"; the book is published in English translation as '''''The Language of the Third Reich'''''.\n''Lingua Tertii Imperii'' studies the way that Nazi propaganda altered the German language to inculcate people with the ideas of Nazism. The book was written under the form of personal notes which Klemperer wrote in his diary, especially from the rise of the Nazi regime in 1933, and even more after 1935, when Klemperer, was stripped of his academic title because he was of Jewish descent. His diary became a notebook in which he noted and commented on the linguistic relativity of the German used by Nazi officials, ordinary citizens, and even fellow Jews. Klemperer wrote the book, based on his notes, in 1945\u20131946."}], "Meaning and Necessity": [{"947 Science Linguistics books": "'''''Meaning and Necessity: A Study in Semantics and Modal Logic''''' (1947; enlarged edition 1956) is a book about semantics and modal logic by the philosopher Rudolf Carnap. The book, in which Carnap discusses the nature of linguistic expressions, was a continuation of his previous work in semantics in ''Introduction to Semantics'' (1942) and ''Formalization of Logic'' (1943)."}]}}} +{"query": "1960s romance films that are also 1960s musical films excluding Musical comedy films", "docs": ["Guitars Sound Softly Through the Night", "Paris Blues", "Naila (film)", "The Umbrellas of Cherbourg", "Lakhon Mein Aik", "Meenda Sorgam", "Heaven Is Never a Great Distance", "Los Tarantos", "Roustabout (film)", "West Side Story (1961 film)", "Thillana Mohanambal", "All the Fine Young Cannibals"], "original_query": "1960s romance films that are also 1960s musical films but not Musical comedy films", "scores": null, "metadata": {"template": "_ that are also _ but not _", "domain": "films", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Guitars Sound Softly Through the Night": ["Definitely relevant"], "Paris Blues": ["Definitely relevant"], "Naila (film)": ["Definitely relevant"], "The Umbrellas of Cherbourg": ["Definitely relevant"], "Lakhon Mein Aik": ["Definitely relevant"], "Meenda Sorgam": ["Definitely relevant"], "Heaven Is Never a Great Distance": ["Likely relevant"], "Los Tarantos": ["Likely relevant"], "Roustabout (film)": ["Likely relevant"], "West Side Story (1961 film)": ["Definitely relevant"], "Thillana Mohanambal": ["Definitely relevant"], "All the Fine Young Cannibals": ["Likely relevant"]}, "evidence_ratings": {"Guitars Sound Softly Through the Night": ["Complete"], "Paris Blues": ["Complete"], "Naila (film)": ["Complete"], "The Umbrellas of Cherbourg": ["Partial"], "Lakhon Mein Aik": ["Complete"], "Meenda Sorgam": ["Complete"], "Heaven Is Never a Great Distance": ["Partial"], "Los Tarantos": ["Partial"], "Roustabout (film)": ["Partial"], "West Side Story (1961 film)": ["Complete"], "Thillana Mohanambal": ["Complete"], "All the Fine Young Cannibals": ["Partial"]}, "attributions": {"Guitars Sound Softly Through the Night": [{"1960s romance films that are also 1960s musical films excluding Musical comedy films": "'''''Guitars Sound Softly Through the Night''''' (German: '''''Gitarren klingen leise durch die Nacht''''') is a 1960 Austrian romantic musical film directed by Hans Deppe and starring Fred Bertelmann, Margit N\u00fcnke and Vivi Bach."}], "Paris Blues": [{"1960s romance films that are also 1960s musical films excluding Musical comedy films": "'''''Paris Blues''''' is a 1961 American musical romantic drama film"}], "Naila (film)": [{"1960s romance films that are also 1960s musical films excluding Musical comedy films": "'''''Naila''''' is a 1965 Pakistani musical romance drama film"}], "The Umbrellas of Cherbourg": [{"1960s romance films that are also 1960s musical films excluding Musical comedy films": "'''''The Umbrellas of Cherbourg''''' () is a 1964 musical romantic drama film written"}], "Lakhon Mein Aik": [{"1960s romance films that are also 1960s musical films excluding Musical comedy films": "'''''Lakhon Mein Aik''''' (; ) is a 1967 Pakistani romantic musical film directed by Raza Mir and written by Zia Sarhadi."}], "Meenda Sorgam": [{"1960s romance films that are also 1960s musical films excluding Musical comedy films": "'''''Meenda Sorgam''''' () is a 1960 Indian Tamil-language romantic musical film"}], "Heaven Is Never a Great Distance": [{"1960s romance films that are also 1960s musical films excluding Musical comedy films": "'''''Heaven Is Never a Great Distance''''' ''(Persian title: Behesht door nist- )'' is a 1969 Iranian ''Persian-genre'' dance Romantic film"}], "Los Tarantos": [{"1960s romance films that are also 1960s musical films": "'''''Los Tarantos''''' is a 1963 Spanish musical drama film", "romance": "The film is based on the play ''La historia de los Tarantos'' written by Alfredo Ma\u00f1as, and inspired by ''Romeo and Juliet'' by William Shakespeare.\nThe love between two gipsies, Juana La Zoronga and Rafael El Taranto, from different families in Barcelona is thwarted by the enmity between their respective parents. Rafael sees Juana dance at a gipsy wedding, and is captivated by her beauty and charm, and they fall in love, aided by their younger siblings who are secretly friends and sympathetic to the young lovers."}], "Roustabout (film)": [{"1960s romance films that are also 1960s": "'''''Roustabout''''' is a 1964 American musical feature film starring Elvis Presley as a singer who takes a job working with a struggling carnival. The film was produced by Hal Wallis and directed by John Rich from a screenplay by Anthony Lawrence and Allan Weiss. The screenplay was nominated for a Writers Guild of America award for best written American musical although ''Roustabout'' received a lukewarm review in ''Variety''. The film's soundtrack album was one of Elvis Presley's most successful, reaching no. 1 on the Billboard Album Chart."}], "West Side Story (1961 film)": [{"1960s romance films that are also 1960s musical films excluding Musical comedy films": "'''''West Side Story''''' is a 1961 American musical romantic drama film directed by Robert Wise and Jerome Robbins."}], "Thillana Mohanambal": [{"1960s musical films excluding Musical comedy films": "'''''Thillana Mohanambal''''' () is a 1968 Indian Tamil-language musical drama film written, directed and produced by A. P. Nagarajan.", "romance films": "It tells the story of Shanmugasundaram, a ''nadaswaram'' player who falls in love with Mohanambal, a ''Bharatanatyam'' dancer who reciprocates his feelings, but unfortunate circumstances and their egoistic nature prevents them from confessing their love for one another."}], "All the Fine Young Cannibals": [{"1960s": "'''''All the Fine Young Cannibals''''' is a 1960 American film directed by Michael Anderson, based on the novel by Rosamond Marshall, starring Robert Wagner, Natalie Wood, Susan Kohner, George Hamilton and Pearl Bailey."}]}}} +{"query": "Non-fiction history books from 2019 about genocide", "docs": ["The Thirty-Year Genocide", "The Volunteer (book)"], "original_query": "2019 non-fiction books that are also History books about genocide", "scores": null, "metadata": {"template": "_ that are also _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"The Thirty-Year Genocide": ["Definitely relevant"], "The Volunteer (book)": ["Definitely relevant"]}, "evidence_ratings": {"The Thirty-Year Genocide": ["Complete"], "The Volunteer (book)": ["Partial"]}, "attributions": {"The Thirty-Year Genocide": [{"Non-fiction history books from 2019 about genocide": "'''The Thirty-Year Genocide: Turkey's Destruction of Its Christian Minorities, 1894\u20131924''''' is a 2019 history book written by Benny Morris and Dror Ze'evi. They argue that the Armenian genocide and other contemporaneous persecution of Christians in the Ottoman Empire constitute an extermination campaign, or genocide, carried out by the Ottoman Empire against its Christian subjects."}], "The Volunteer (book)": [{"Non-fiction history books from 2019": "'''''The Volunteer: The True Story of the Resistance Hero Who Infiltrated Auschwitz''''' (British title; the American edition is titled '''''The Volunteer: One Man's Mission to Lead an Underground Army Inside Auschwitz and Stop the Holocaust''''') is a 2019 book which presents research by British writer Jack Fairweather, a former ''Washington Post'' war correspondent, into the life of Witold Pilecki, a Polish soldier and Home Army resistance fighter who infiltrated the infamous Auschwitz concentration camp."}]}}} +{"query": "cultural geography and Science books but not about creativity", "docs": ["The China\u2013Pakistan Axis", "Clash of Civilizations", "The Nine Nations of North America", "When Work Disappears", "Jewish Roots in Ukraine and Moldova", "Hellenic Nomarchy", "The End of History and the Last Man", "Searching for Whitopia", "Where Once We Walked", "The Journey of Man", "International Encyclopedia of Human Geography", "The Real Eve", "Geographia Neoteriki", "The Wealth and Poverty of Nations", "Jewish Roots in Poland"], "original_query": "Science books that are also Books about cultural geography but not Books about creativity", "scores": null, "metadata": {"template": "_ that are also _ but not _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"The China\u2013Pakistan Axis": ["Definitely relevant"], "Clash of Civilizations": ["Likely relevant"], "The Nine Nations of North America": ["Likely relevant"], "When Work Disappears": ["Likely relevant"], "Jewish Roots in Ukraine and Moldova": ["Definitely relevant"], "Hellenic Nomarchy": ["Likely relevant"], "The End of History and the Last Man": ["Definitely relevant"], "Searching for Whitopia": ["Likely relevant"], "Where Once We Walked": ["Likely relevant"], "The Journey of Man": ["Definitely relevant"], "International Encyclopedia of Human Geography": ["Definitely relevant"], "The Real Eve": ["Definitely relevant"], "Geographia Neoteriki": ["Definitely relevant"], "The Wealth and Poverty of Nations": ["Definitely relevant"], "Jewish Roots in Poland": ["Likely relevant"]}, "evidence_ratings": {"The China\u2013Pakistan Axis": ["Complete"], "Clash of Civilizations": ["Complete"], "The Nine Nations of North America": ["Partial"], "When Work Disappears": ["Complete"], "Jewish Roots in Ukraine and Moldova": ["Partial"], "Hellenic Nomarchy": ["Partial"], "The End of History and the Last Man": ["Complete"], "Searching for Whitopia": ["Partial"], "Where Once We Walked": ["Partial"], "The Journey of Man": ["Complete"], "International Encyclopedia of Human Geography": ["Complete"], "The Real Eve": ["Complete"], "Geographia Neoteriki": ["Complete"], "The Wealth and Poverty of Nations": ["Complete"], "Jewish Roots in Poland": ["Partial"]}, "attributions": {"The China\u2013Pakistan Axis": [{"cultural geography": "'''''The China\u2013Pakistan Axis: Asia's New Geopolitics''''' is a book by the British author Andrew Small exploring the cultural and political ties as well as depicting the relationship between Pakistan and China, released in early 2015."}], "Clash of Civilizations": [{"cultural geography and Science books": "The '''Clash of Civilizations''' is a thesis that people's cultural and religious identities will be the primary source of conflict in the post\u2013Cold War world. The American political scientist Samuel P. Huntington argued that future wars would be fought not between countries, but between cultures."}], "The Nine Nations of North America": [{"cultural geography": "'''''The Nine Nations of North America''''' is a 1981 book by Joel Garreau, in which the author suggests that North America can be divided into nine nations, which have distinctive economic and cultural features."}], "When Work Disappears": [{"cultural geography": "Wilson's argument is that the disappearance of work and the consequences of that disappearance for both social and cultural life are the central problems in the inner-city ghetto.", "Science books": "Wilson's book uses evidence from large-scale scientific surveys in the ghetto and information culled from ethnographic interviews of ghetto residents in order to create a complete picture of the problems that face the residents."}], "Jewish Roots in Ukraine and Moldova": [{"cultural geography and Science books but not about creativity": "'Jewish Roots in Ukraine and Moldova''''' (full title: ''Jewish Roots in Ukraine and Moldova: Pages from the Past and Archival Inventories'') is a book created by genealogist Miriam Weiner and co-published by The Miriam Weiner Routes to Roots Foundation and YIVO Institute for Jewish Research. A searchable database of updated archival holdings listed in the book"}], "Hellenic Nomarchy": [{"cultural geography": "It advocated the ideals of freedom, social justice and equality as the main principles of a well-governed society, making it the most important theoretical monument of Greek republicanism."}], "The End of History and the Last Man": [{"cultural geography and Science books but not about creativity": "who define human history as a linear progression, from one socioeconomic epoch to another."}], "Searching for Whitopia": [{"cultural geography": "African American journalist Rich Benjamin documents his journeys to find out why more and more white Americans move to small towns and areas that are, for the most part, white, and to explain why Whitopias are growing and what it means for the United States."}], "Where Once We Walked": [{"cultural geography and Science books but not about creativity": "'''''Where Once We Walked''''' (full title: ''Where Once We Walked: A Guide to the Jewish Communities Destroyed in The Holocaust''), compiled by noted genealogist Gary Mokotoff and Sallyann Amdur Sack with Alexander Sharon, is a gazetteer of 37,000 town names in Central and Eastern Europe focusing on those with Jewish populations in the 19th and first half of the 20th centuries and most of whose Jewish communities were almost or completely destroyed during The Holocaust."}], "The Journey of Man": [{"cultural geography and Science books but not about creativity": "American geneticist and anthropologist, in which he uses techniques and theories of genetics and evolutionary biology to trace the geographical dispersal of early human migrations out of Africa"}], "International Encyclopedia of Human Geography": [{"cultural geography and Science books": "The '''''International Encyclopedia of Human Geography''''' is a 2009 academic reference work covering human geography."}], "The Real Eve": [{"cultural geography and Science books": "'''''The Real Eve: Modern Man's Journey Out of Africa''''' is a popular science book about the evolution of modern humans written by British geneticist Stephen Oppenheimer."}], "Geographia Neoteriki": [{"cultural geography and Science books but not about creativity": "'''''Geographia Neoteriki''''' ( ''Modern Geography'') is a geography book written in Greek by Daniel Philippidis and Grigorios Konstantas and printed in Vienna in 1791. It focused on both the physical and human geography features of the European continent and especially on Southeastern Europe, and is considered one of the most remarkable works of the modern Greek Enlightenment."}], "The Wealth and Poverty of Nations": [{"cultural geography and Science books": "Landes attempted to explain why some countries and regions experienced near miraculous periods of explosive growth while the rest of the world stagnated. The book compared the long-term economic histories of different regions, specifically Europe, United States, Japan, China, the Arab world, and Latin America. In addition to analyzing economic and cliometric figures, he credited intangible assets, such as culture and enterprise, to explain economic success or failure.", "but not about creativity": "In addition to analyzing economic and cliometric figures, he credited intangible assets, such as culture and enterprise, to explain economic success or failure."}], "Jewish Roots in Poland": [{"cultural geography": "The book includes archival holdings of the Polish State Archives, the Jewish Historical Institute in Warsaw, local town hall documents throughout Poland, Holocaust documents found in the archives of the death camps located in Auschwitz near Krakow and Majdanek near Lublin. The book also features document examples, maps, antique postcards depicting towns and daily life, and modern-day photographs."}]}}} +{"query": "2010s non-fiction books that are about social psychology but not crowd psychology", "docs": ["The Boy in the River", "Silent No More (book)", "A Very English Scandal", "Simple Choices", "A History of Marriage", "Quiet Power", "The End of American Childhood", "IGen (book)", "Scarcity: Why Having Too Little Means So Much", "The Argonauts", "The Righteous Mind", "Battle Hymn of the Tiger Mother", "The New Childhood", "Far from the Tree", "The Importance of Being Little", "The Coddling of the American Mind"], "original_query": "2010s non-fiction books that are also Books about social psychology but not Books about crowd psychology", "scores": null, "metadata": {"template": "_ that are also _ but not _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"The Boy in the River": ["Likely relevant"], "Silent No More (book)": ["Definitely relevant"], "A Very English Scandal": ["Likely relevant"], "Simple Choices": ["Definitely relevant"], "A History of Marriage": ["Likely relevant"], "Quiet Power": ["Likely relevant"], "The End of American Childhood": ["Likely relevant"], "IGen (book)": ["Definitely relevant", "Likely not relevant"], "Scarcity: Why Having Too Little Means So Much": ["Definitely relevant"], "The Argonauts": ["Likely relevant"], "The Righteous Mind": ["Definitely relevant"], "Battle Hymn of the Tiger Mother": ["Likely relevant"], "The New Childhood": ["Likely relevant"], "Far from the Tree": ["Definitely relevant"], "The Importance of Being Little": ["Likely relevant"], "The Coddling of the American Mind": ["Likely relevant"]}, "evidence_ratings": {"The Boy in the River": ["Partial"], "Silent No More (book)": ["Partial"], "A Very English Scandal": ["Partial"], "Simple Choices": ["Complete"], "A History of Marriage": ["Partial"], "Quiet Power": ["Partial"], "The End of American Childhood": ["Partial"], "IGen (book)": ["Complete", "Complete"], "Scarcity: Why Having Too Little Means So Much": ["Complete"], "The Argonauts": ["Partial"], "The Righteous Mind": ["Complete"], "Battle Hymn of the Tiger Mother": ["Partial"], "The New Childhood": ["Partial"], "Far from the Tree": ["Complete"], "The Importance of Being Little": ["Partial"], "The Coddling of the American Mind": ["Partial"]}, "attributions": {"The Boy in the River": [{"2010s non-fiction books that are about social psychology": "'''''The Boy in the River''''' is a 2012 book by the British author and criminologist Richard Hoskins about the 'Adam' murder case."}], "Silent No More (book)": [{"2010s non-fiction books that are about social psychology but not crowd psychology": "'''''Silent No More: Victim 1's Fight for Justice Against Jerry Sandusky''''' is a 2012 book by Aaron Fisher, identified as \"Victim 1\" in the Penn State child sex abuse scandal. Fisher is called \"Victim 1\" because it was his reporting his abuse to high school officials that set off the investigation that led to Sandusky's conviction. The book follows Fisher's experience from the beginning of his interaction with Jerry Sandusky at The Second Mile through Sandusky's conviction of 45 of 48 counts related to child sex abuse."}], "A Very English Scandal": [{"non-fiction books": "'''''A Very English Scandal''''' is a true crime non-fiction novel by John Preston.", "2010s": "It was first published on 5 May 2016 by Viking Press and by Other Press in the United States."}], "Simple Choices": [{"2010s non-fiction books that are about social psychology but not crowd psychology": "'''''Simple Choices: Thoughts on choosing environments that support who your child is meant to be''''' is a parenting book by Lisa Graham Keegan published in 2013.\n\n\n''Simple Choices'' begins with Keegan's memoir of building a family through divorce, abandonment, adoption, mental impairment, sexual identity and all the challenges life has to offer."}], "A History of Marriage": [{"2010s non-fiction books that are about social psychology but not crowd psychology": "'''''A History of Marriage''''', published by Seven Stories Press, is a non-fiction book by Elizabeth Abbott, the Canadian author of ''A History of Celibacy'' (1999) and ''A History of Mistresses'' (2003) that combines general history and personal histories of marriage. The book is a study of mostly North American rituals of courting, nuptials, marriage, sex, child-raising and divorce. Some topics covered are relative ages at which various societies from Chinese to Mormon married off their girls; details of the satisfying marriage of Martin Luther and former nun Katharina von Bora; the ruptured family units of Native American children removed to residential schools; the popularity of so-called Boston marriages (depicted by Henry James in ''The Bostonians'') between like-minded women who resisted conventional marriage but weren't necessarily lesbian; and the scarcity of sponges used for contraception by Northern women during the Civil War because of the cut-off in supply from Florida. ''A History of Marriage'' was a finalist for the 2010 Governor General's Literary Award for non-fiction.\n\n\nA paperback edition was published in August 2015 by Seven Stories Press."}], "Quiet Power": [{"2010s non-fiction books that are about social psychology but not crowd psychology": "'''''Quiet Power: The Secret Strengths of Introverts''''' is a 2016 non-fiction book written by Susan Cain with Gregory Mone and Erica Moroz, and illustrated by Grant Snider.", "social psychology but": "''Quiet Power'' discusses the distinction between introversion and shyness; deeper student engagement versus conventional expectations of class participation; speaking in front of groups; individual versus group work; introvert-friendly methods of structuring group work; and use of social media in education. The book focuses on shyness in addition to introversion, saying that shyness involves fear of social judgment."}], "The End of American Childhood": [{"2010s non-fiction books that are about social psychology": "'''''The End of American Childhood: A History of Parenting from Life on the Frontier to the Managed Child''''' was written by historian Paula S. Fass and published by Princeton University Press in 2016."}], "IGen (book)": [{"2010s non-fiction books that are about social psychology but not crowd psychology": "'''''iGen: Why Today's Super-Connected Kids Are Growing Up Less Rebellious, More Tolerant, Less Happy\u2014and Completely Unprepared for Adulthood\u2014and What That Means for the Rest of Us''''' is a 2017 nonfiction book by Jean Twenge which studies the lifestyles, habits and values of Americans born 1995\u20132012, the first generation to reach adolescence after smartphones became widespread."}, null], "Scarcity: Why Having Too Little Means So Much": [{"2010s non-fiction books that are about social psychology but not crowd psychology": "'''''Scarcity: Why Having Too Little Means So Much''''' is a 2013 book by behavioural economist Sendhil Mullainathan and psychologist Eldar Shafir. The authors discuss the role of scarcity in creating, perpetuating, and alleviating poverty. The book also proposes several ideas for how individuals and groups of people can handle scarcity to achieve success and satisfaction."}], "The Argonauts": [{"2010s": "'''''The Argonauts''''' is a book by poet and critic Maggie Nelson, published in 2015", "social psychology": "The book discusses her romantic relationship with the transgender artist Harry Dodge leading to her pregnancy as well as topics ranging from the death of a parent, transgender embodiment, academia, familial relationships, and the limitations of language."}], "The Righteous Mind": [{"2010s non-fiction books that are about social psychology but not crowd psychology": "A simple graphic depicting survey data from the United States intended to support moral foundations theory.\n'''''The Righteous Mind: Why Good People are Divided by Politics and Religion''''' is a 2012 social psychology book by Jonathan Haidt, in which the author describes human morality as it relates to politics and religion.\nIn the first part of the book, the author uses research to demonstrate social intuitionism, how people's beliefs come primarily from their intuitions, and rational thought often comes after to justify initial beliefs. He cites David Hume and E. O. Wilson as thinkers who gave reason a relatively low estimation, as opposed to more popular thinkers who give reason a central place in moral cognition, such as Lawrence Kohlberg and his stages of moral reasoning."}], "Battle Hymn of the Tiger Mother": [{"2010s non-fiction books that are about social psychology": "'''''Battle Hymn of the Tiger Mother''''' is a book by American author and law professor Amy Chua that was published in 2011."}], "The New Childhood": [{"2010s non-fiction books that are about social psychology": "in December 2018.\nShapiro, who teaches at Temple University in the College of Liberal Arts, became interested in childhood development and digital play following his divorce, after playing video games with his two sons. He argues against strict screen time limits for kids, suggesting instead that parents should teach their kids how to use digital devices with integrity."}], "Far from the Tree": [{"2010s non-fiction books that are about social psychology": "'''''Far From the Tree: Parents, Children, and the Search for Identity''''' is a non-fiction book by Andrew Solomon published in November 2012 in the United States and two months later in the UK (under the title, ''Far from the Tree: A Dozen Kinds of Love''), about how families accommodate children with physical, mental and social disabilities and differences."}], "The Importance of Being Little": [{"2010s non-fiction books that are about social psychology but not crowd psychology": "'''''The Importance of Being Little: What Preschoolers Really Need from Grownups''''' is a 2016 book written by early childhood educator Erika Christakis that argues the importance of separating childhood from adulthood. It was published by Viking Press."}], "The Coddling of the American Mind": [{"2010s": "2018 book"}]}}} +{"query": "American novels about New Mexico from the 1980s", "docs": ["Talking God", "The Dark Wind", "A Thief of Time", "People of Darkness", "Contact (novel)", "The Cardinal of the Kremlin", "Skinwalkers (novel)", "Tiger Eyes"], "original_query": "1980s books that are also both American novels and Books about New Mexico", "scores": null, "metadata": {"template": "_ that are also both _ and _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Talking God": ["Likely relevant"], "The Dark Wind": ["Definitely relevant"], "A Thief of Time": ["Definitely relevant"], "People of Darkness": ["Definitely relevant"], "Contact (novel)": ["Definitely relevant"], "The Cardinal of the Kremlin": ["Likely relevant"], "Skinwalkers (novel)": ["Definitely relevant"], "Tiger Eyes": ["Definitely relevant"]}, "evidence_ratings": {"Talking God": ["Partial"], "The Dark Wind": ["Complete"], "A Thief of Time": ["Complete"], "People of Darkness": ["Partial"], "Contact (novel)": ["Complete"], "The Cardinal of the Kremlin": ["Partial"], "Skinwalkers (novel)": ["Partial"], "Tiger Eyes": ["Complete"]}, "attributions": {"Talking God": [{"American novels about New Mexico from the 1980s": "An unidentified man is found dead along the railroad tracks near Gallup, New Mexico."}], "The Dark Wind": [{"American novels about New Mexico from the 1980s": "'''The Dark Wind''''' is the fifth crime fiction novel in the Joe Leaphorn / Jim Chee Navajo Tribal Police series by Tony Hillerman, published in 1982. It is the second of the novels to feature Officer Jim Chee."}], "A Thief of Time": [{"from the 1980s": "''''A Thief of Time''''' is the eighth crime fiction novel Joe Leaphorn / Jim Chee Navajo Tribal Police series by Tony Hillerman, first published in 1988.", "about New Mexico": "#Albuquerque, NM\n\n#Aztec, NM\n#Bisti (Badlands and Trading Post), NM\n\n#Bitani Tsosi Wash, NM\n\n#Blanco, NM", "American novels": "'''''A Thief of Time''''' is the eighth crime fiction novel Joe Leaphorn / Jim Chee Navajo Tribal Police series by Tony Hillerman, first published in 1988."}], "People of Darkness": [{"American novels about New Mexico from the 1980s": "'''''People of Darkness''''' is the fourth crime fiction novel in the Joe Leaphorn / Jim Chee Navajo Tribal Police series by author Tony Hillerman, first published in 1980."}], "Contact (novel)": [{"American novels": "'''''Contact''''' is a 1985 hard science fiction novel by American scientist Carl Sagan.", "from the 1980s": "'''''Contact''''' is a 1985 hard science fiction novel by American scientist Carl Sagan.", "about New Mexico": "She eventually becomes the director of \"Project Argus\", a radiotelescope array in New Mexico dedicated to the search for extraterrestrial intelligence (SETI)."}], "The Cardinal of the Kremlin": [{"about New Mexico": "The mole, a lesbian named Dr. Beatrice Taussig who unluckily falls in love with Gregory's fianc\u00e9e, eventually gives up Bisyarina to the FBI out of guilt, and the Hostage Rescue Team later saves Gregory from his Soviet captors in a shabby desert safe house in New Mexico.", "from the 1980s": "'''''The Cardinal of the Kremlin''''' is an espionage thriller novel, written by Tom Clancy and released on May 20, 1988.", "American": "Ryan, who is part of the American arms negotiation team, travels to Moscow for the arms reduction talks."}], "Skinwalkers (novel)": [{"American novels about New Mexico from the 1980s": "'''''Skinwalkers''''' is the seventh crime-fiction novel in the Joe Leaphorn/Jim Chee Navajo Tribal Police series by author Tony Hillerman published in 1986. The film version,''Skinwalkers'', was adapted for television for the PBS ''Mystery!'' series in 2002."}], "Tiger Eyes": [{"American novels about New Mexico from the 1980s": "'''''Tiger Eyes''''' is a young adult novel written by Judy Blume in 1981 about a 15-year-old girl attempting to cope with the unexpected death of her father. In 2012, the novel was adapted into a film of the same name, directed by Judy's son, Lawrence Blume, and starring Willa Holland as Davey Wexler. Davey is often sad, depressed and defensive on everything. She has a short breakdown at her brother, Jason. Her mom becomes incapable of doing things about the house, making Davey more depressed.\nDavis \u201cDavey\u201d Wexler, along with her mother, Gwen, and her little brother, Jason, have just attended the funeral of her father, Adam, who was shot to death in a holdup at their 7-Eleven convenience store in Atlantic City. After lying in bed for days on end and not eating, Davey starts her tenth year of school, but faints\non her first day from anxiety. She goes for a checkup, and the doctor explains Davey is having panic attacks.\n\n\nDavey's mother, Gwen, decides they need to get away for awhile and takes up an offer from Adam's older sister, Bitsy, and his brother-in-law Walter to come stay with them in Los Alamos, New Mexico. A few days before they are scheduled to return to Atlantic City, Gwen receives news their store has been further vandalized, and she decides they're going to stay in Los Alamos through the end of the school year."}]}}} +{"query": "Fauna of Tibet and Arthropods of Asia", "docs": ["Spilosoma erythrozona", "Spilarctia leopardina", "Scorpiops langxian", "Areas galactina", "Bhutanitis thaidina", "Scorpiops tibetanus", "Mangina argus", "Callindra equitalis", "Sinopieris dubernardi", "Callindra principalis", "Eupithecia magica", "Ypthima sakra", "Athyma opalina", "Spilarctia nydia", "Rhyzodiastes orestes"], "original_query": "Arthropods of Asia that are also Fauna of Tibet", "scores": null, "metadata": {"template": "_ that are also _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "animals"}} +{"query": "Endemic fauna of Japan and Invertebrates of Asia but not Lepidoptera of Asia", "docs": ["Nebria pusilla", "Galloisiana nipponensis", "Agelena babai", "Galloisiana yuasai", "Galloisiana notabilis", "Clinidium veneficum", "Asemonea tanikawai", "Enteroplax yaeyamensis", "Sinopoda okinawana", "Galloisiana kiyosawai", "Calopteryx cornelia", "Parafontaria", "Galloisiana yezoensis", "Camponotus bishamon", "Cambaroides japonicus", "Boninena callistoderma", "Ogasawarana yoshiwarana", "Hirasea acutissima"], "original_query": "Invertebrates of Asia that are also Endemic fauna of Japan but not Lepidoptera of Asia", "scores": null, "metadata": {"template": "_ that are also _ but not _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "animals"}} +{"query": "Birds found in Sierra Madre Oriental that aren't found in Sierra Madre Occidental", "docs": ["Amethyst-throated mountaingem", "Red-lored amazon", "Gartered trogon", "Spot-crowned woodcreeper", "Northern bobwhite", "Great curassow", "Maroon-fronted parrot", "Curve-winged sabrewing", "Crimson-collared grosbeak", "Greenish elaenia", "White-faced quail-dove", "Black thrush", "Singing quail", "Crested guan", "Blue-capped motmot", "Bearded wood partridge", "Golden-fronted woodpecker", "Tamaulipas pygmy owl"], "original_query": "Birds of the Sierra Madre Oriental that are not Birds of the Sierra Madre Occidental", "scores": null, "metadata": {"template": "_ that are not _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "animals"}} +{"query": "1992 non-American independent films", "docs": ["Fortress (1992 film)", "Man Bites Dog (film)", "The Colours of My Father: A Portrait of Sam Borenstein", "The Bridge (1992 film)", "The Little Punker", "A Place in the World (film)", "The Crying Game", "Romper Stomper", "Star Time (film)", "The Last Days of Chez Nous", "Swoon (film)"], "original_query": "1992 independent films that are not American independent films", "scores": null, "metadata": {"template": "_ that are not _", "domain": "films", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Fortress (1992 film)": ["Likely relevant"], "Man Bites Dog (film)": ["Definitely relevant"], "The Colours of My Father: A Portrait of Sam Borenstein": ["Likely relevant"], "The Bridge (1992 film)": ["Likely relevant"], "The Little Punker": ["Definitely relevant"], "A Place in the World (film)": ["Likely relevant"], "The Crying Game": ["Likely relevant"], "Romper Stomper": ["Likely relevant"], "Star Time (film)": ["Definitely relevant"], "The Last Days of Chez Nous": ["Likely relevant"], "Swoon (film)": ["Likely relevant"]}, "evidence_ratings": {"Fortress (1992 film)": ["Partial"], "Man Bites Dog (film)": ["Complete"], "The Colours of My Father: A Portrait of Sam Borenstein": ["Partial"], "The Bridge (1992 film)": ["Complete"], "The Little Punker": ["Complete"], "A Place in the World (film)": ["Complete"], "The Crying Game": ["Partial"], "Romper Stomper": ["Partial"], "Star Time (film)": ["No evidence"], "The Last Days of Chez Nous": ["Partial"], "Swoon (film)": ["Partial"]}, "attributions": {"Fortress (1992 film)": [{"1992 non-American independent films": "'''''Fortress''''' is a 1992 science fiction action film"}], "Man Bites Dog (film)": [{"independent films": "''Man Bites Dog'' is shot in black and white on 16mm film and was produced on a shoe-string budget by four student filmmakers, led by director R\u00e9my Belvaux.", "1992 non-American": "'''''Man Bites Dog''''' (, literally \"It Happened Near Your Home\") is a 1992 Belgian black comedy crime mockumentary written, produced and directed by R\u00e9my Belvaux, Andr\u00e9 Bonzel and Beno\u00eet Poelvoorde, who are also the film's co-editor, cinematographer and lead actor respectively."}], "The Colours of My Father: A Portrait of Sam Borenstein": [{"1992 non-American": "'''''The Colours of My Father: A Portrait of Sam Borenstein''''' is a 1992 Canadian short animated documentary film directed by Joyce Borenstein."}], "The Bridge (1992 film)": [{"1992 non-American independent films": "a 1992 independent film", "non-American": "Directed by Sydney Macartney, it stars Saskia Reeves"}], "The Little Punker": [{"1992 non-American independent films": "a 1992 German animated feature independent dramedy film"}], "A Place in the World (film)": [{"1992 non-American independent films": "a 1992 Argentine drama film co-written, co-produced and directed by Adolfo Aristarain, and starring Federico Luppi."}], "The Crying Game": [{"1992 non-American": "'''''The Crying Game''''' is a 1992 thriller film written and directed by Neil Jordan, produced by Stephen Woolley, and starring Stephen Rea, Miranda Richardson, Jaye Davidson, Adrian Dunbar, Ralph Brown, and Forest Whitaker. The film explores themes of race, sex, nationality, and sexuality against the backdrop of the Troubles in Northern Ireland."}], "Romper Stomper": [{"1992 non-American independent films": "'''''Romper Stomper''''' is a 1992 Australian drama film"}], "Star Time (film)": [null], "The Last Days of Chez Nous": [{"1992 non-American": "'''''The Last Days of Chez Nous''''' is a 1992 Australian drama film directed by Gillian Armstrong and written by Helen Garner."}], "Swoon (film)": [{"1992 non-American independent films": "'''''Swoon''''' is a 1992 independent film written and directed by Tom Kalin."}]}}} +{"query": "Non Horror demon novels.", "docs": ["List of the Lost", "Blood Price", "The Black Spider", "The Castle in the Forest", "The Devil in Love (novel)", "Melmoth the Wanderer", "Practical Demonkeeping", "Artemis Fowl and the Lost Colony", "The Black Tattoo", "Good Omens", "Eric (novel)"], "original_query": "Demon novels that are not Horror novel series", "scores": null, "metadata": {"template": "_ that are not _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"List of the Lost": ["Definitely relevant"], "Blood Price": ["Definitely relevant"], "The Black Spider": ["Likely relevant"], "The Castle in the Forest": ["Likely relevant"], "The Devil in Love (novel)": ["Definitely relevant"], "Melmoth the Wanderer": ["Definitely relevant"], "Practical Demonkeeping": ["Likely relevant"], "Artemis Fowl and the Lost Colony": ["Definitely relevant"], "The Black Tattoo": ["Definitely relevant"], "Good Omens": ["Likely relevant"], "Eric (novel)": ["Definitely relevant"]}, "evidence_ratings": {"List of the Lost": ["Complete"], "Blood Price": ["Complete"], "The Black Spider": ["Partial"], "The Castle in the Forest": ["Partial"], "The Devil in Love (novel)": ["Complete"], "Melmoth the Wanderer": ["Complete"], "Practical Demonkeeping": ["Partial"], "Artemis Fowl and the Lost Colony": ["Complete"], "The Black Tattoo": ["Complete"], "Good Omens": ["Partial"], "Eric (novel)": ["Complete"]}, "attributions": {"List of the Lost": [{"Non Horror demon novels.": "The book is about a 1970s relay team in Boston who accidentally kill a homeless person, whose death brings misfortune to the team."}], "Blood Price": [{"Non Horror demon novels.": "He tells her that the killer is a demon, that she actually did see him disappear."}], "The Black Spider": [{"demon": "The hunter used his demonic powers to instill a curse in the kiss, which would ensure his payment."}], "The Castle in the Forest": [{"Non Horror demon novels.": "'''''The Castle in the Forest''''' is the last novel by writer Norman Mailer, published in the year of his death, 2007. It is the story of Adolf Hitler's childhood as seen through the eyes of Dieter, a demon sent to put him on his destructive path. The novel explores the idea that Hitler was the product of incest. It forms a thematic contrast with the writer's immediately previous novel ''The Gospel According to the Son'' (1999), which deals with the early life of Jesus. It received a good deal of praise, including a glowing review from Lee Siegel of ''The New York Times Book Review'', and was the ''New York Times'' Bestseller for 2007."}], "The Devil in Love (novel)": [{"Non Horror demon novels.": "Author of ''The Devil in Love, Jacques Cazotte''\n'''''The Devil in Love''''' (, 1772) is an occult romance by Jacques Cazotte which tells of a demon, or devil, who falls in love with a young Spanish nobleman named Don Alvaro, an amateur human dabbler, and attempts, in the guise of a young woman, to win his affections."}], "Melmoth the Wanderer": [{"Non Horror demon novels.": "'''''Melmoth the Wanderer''''' is an 1820 Gothic novel by Irish playwright, novelist and clergyman Charles Maturin. The novel's titular character is a scholar who sold his soul to the devil in exchange for 150 extra years of life, and searches the world for someone who will take over the pact for him, in a manner reminiscent of the Wandering Jew."}], "Practical Demonkeeping": [{"Non Horror demon novels": "His first novel, it deals with a demon from Hell and his master."}], "Artemis Fowl and the Lost Colony": [{"Non Horror demon novels.": "In Barcelona, Spain, Artemis Fowl II and Butler, his bodyguard, wait for a demon. They suddenly encounter a demon who transports Artemis through time."}], "The Black Tattoo": [{"Non Horror demon novels.": "'''''The Black Tattoo''''' is a young adult fantasy novel by Sam Enthoven, published in 2006. It deals with a boy, Charlie, becoming possessed by a demon that manifests itself in the form of a black tattoo on his body."}], "Good Omens": [{"demon novels.": "There are attempts by the angel Aziraphale and the demon Crowley to sabotage the coming of the end times, having grown accustomed to their comfortable surroundings in England."}], "Eric (novel)": [{"demon novels.": "the Demon King"}]}}} \ No newline at end of file diff --git a/baseline_quest/data/train_subset_nonunion.jsonl b/baseline_quest/data/train_subset_nonunion.jsonl new file mode 100644 index 0000000..59a0ee4 --- /dev/null +++ b/baseline_quest/data/train_subset_nonunion.jsonl @@ -0,0 +1,61 @@ +{"query": "Garden plants that are from both Asia, Australasia and Malesia.", "docs": ["Asplenium nidus", "Cordyline fruticosa"], "original_query": "Garden plants of Asia that are also both Garden plants of Australasia and Flora of Malesia", "scores": null, "metadata": {"template": "_ that are also both _ and _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "plants"}} +{"query": "1992 American children's comedy animated musical films", "docs": ["FernGully: The Last Rainforest", "Aladdin (1992 Disney film)", "Tom and Jerry: The Movie"], "original_query": "1992 films that are also both Animated musical films and American children's comedy films", "scores": null, "metadata": {"template": "_ that are also both _ and _", "domain": "films", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"FernGully: The Last Rainforest": ["Likely relevant"], "Aladdin (1992 Disney film)": ["Definitely relevant"], "Tom and Jerry: The Movie": ["Definitely relevant"]}, "evidence_ratings": {"FernGully: The Last Rainforest": ["Partial"], "Aladdin (1992 Disney film)": ["Complete"], "Tom and Jerry: The Movie": ["Complete"]}, "attributions": {"FernGully: The Last Rainforest": [{"1992": "'''''FernGully: The Last Rainforest''''' is a 1992 animated musical fantasy film, directed by Bill Kroyer and scripted by Jim Cox.", "children's": "'''''FernGully: The Last Rainforest''''' is a 1992 animated musical fantasy film, directed by Bill Kroyer and scripted by Jim Cox", "animated musical films": "'''''FernGully: The Last Rainforest''''' is a 1992 animated musical fantasy film, directed by Bill Kroyer and scripted by Jim Cox.", "American": "Adapted from the book of the same name by Diana Young, the film is an Australian and American venture produced by Kroyer Films, Inc., Youngheart Productions, FAI Films and 20th Century Fox."}], "Aladdin (1992 Disney film)": [{"1992 American children's comedy animated musical films": "'''''Aladdin''''' is a 1992 American animated musical fantasy comedy film produced by Walt Disney Feature Animation and released by Walt Disney Pictures. The film is the 31st Disney animated feature film and was the fourth produced during the Disney Renaissance. It was produced and directed by Ron Clements and John Musker, and is based on the Arabic folktale of the same name from the ''One Thousand and One Nights''. The voice cast features Scott Weinger, Robin Williams, Linda Larkin, Jonathan Freeman, Frank Welker, Gilbert Gottfried, and Douglas Seale. The film follows the titular Aladdin, an Arabian street urchin, who finds a magic lamp containing a genie. He disguises himself as a wealthy prince and tries to impress the Sultan in order to marry his free-spirited daughter, Princess Jasmine."}], "Tom and Jerry: The Movie": [{"1992 American children's comedy animated musical films": "'''''Tom and Jerry: The Movie''''' is a 1992 American animated musical comedy film based on the characters ''Tom and Jerry'' created by William Hanna and Joseph Barbera."}]}}} +{"query": "Spy novels about North America and the military", "docs": ["The Death Strain", "A Matter of Time (Cook novel)", "The Man from Barbarossa", "The Cardinal of the Kremlin", "The Sympathizer", "Camp X (novel)", "The Man Next Door (novel)", "Mongoose R.I.P.", "The Betrayers", "The Man with the Golden Gun (novel)", "Kolymsky Heights", "A Tract of Time"], "original_query": "Books about North America that are also both Novels about the military and Spy novels", "scores": null, "metadata": {"template": "_ that are also both _ and _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"The Death Strain": ["Definitely relevant"], "A Matter of Time (Cook novel)": ["Definitely relevant"], "The Man from Barbarossa": ["Likely relevant"], "The Cardinal of the Kremlin": ["Definitely relevant"], "The Sympathizer": ["Definitely relevant"], "Camp X (novel)": ["Definitely relevant"], "The Man Next Door (novel)": ["Definitely relevant"], "Mongoose R.I.P.": ["Definitely relevant"], "The Betrayers": ["Likely relevant"], "The Man with the Golden Gun (novel)": ["Definitely relevant"], "Kolymsky Heights": ["Definitely relevant"], "A Tract of Time": ["Definitely relevant"]}, "evidence_ratings": {"The Death Strain": ["Complete"], "A Matter of Time (Cook novel)": ["Complete"], "The Man from Barbarossa": ["Partial"], "The Cardinal of the Kremlin": ["Complete"], "The Sympathizer": ["Complete"], "Camp X (novel)": ["Complete"], "The Man Next Door (novel)": ["Partial"], "Mongoose R.I.P.": ["Complete"], "The Betrayers": ["Partial"], "The Man with the Golden Gun (novel)": ["Complete"], "Kolymsky Heights": ["Complete"], "A Tract of Time": ["Complete"]}, "attributions": {"The Death Strain": [{"Spy novels about North America and the military": "Carter is a US secret agent, code-named N-3, with the rank of Killmaster."}], "A Matter of Time (Cook novel)": [{"Spy novels about North America and the military": "'''''A Matter of Time''''' is a novel by Glen Cook, combining elements of science fiction (specifically, time travel), crime fiction and spy thriller. In regard to the last, the novel in particular takes up and expands the theme of American prisoners of war being brainwashed in Communist China and their loyalties reversed \u2013 a theme made famous through the novel ''The Manchurian Candidate'' and film made on its basis."}], "The Man from Barbarossa": [{"Spy novels about North America and the military": "'''''The Man from Barbarossa''''', first published in 1991, was the eleventh novel by John Gardner featuring Ian Fleming's secret agent, James Bond"}], "The Cardinal of the Kremlin": [{"Spy novels": "'''''The Cardinal of the Kremlin''''' is an espionage thriller novel, written by Tom Clancy and released on May 20, 1988.", "the military": "A direct sequel to ''The Hunt for Red October'' (1984), it features CIA analyst Jack Ryan as he extracts CARDINAL, the agency's highest placed agent in the Soviet government who is being pursued by the KGB, as well as the Soviet intelligence agency's director."}], "The Sympathizer": [{"Spy novels about North America and the military": "'''''The Sympathizer''''' is the 2015 debut novel by Vietnamese-American professor Viet Thanh Nguyen. It is a best-selling novel and recipient of the 2016 Pulitzer Prize for Fiction. The novel received generally positive acclaim from critics, and it was named a ''New York Times'' Editor's Choice.\n\n\nThe novel incorporates elements from a number of different novel genres: immigrant, mystery, political, metafiction, dark comedic, historical, spy, and war."}], "Camp X (novel)": [{"Spy novels about North America and the military": "'''''Camp X''''' is a children's spy novel written by Canadian author Eric Walters. Set in World War II, the novel is about the two brothers Jack and George, trying to save a top-secret Canadian military base called Camp X. They accidentally discovered it after playing a fake game of war."}], "The Man Next Door (novel)": [{"Spy novels about North America and the military": "Smith is obviously an enemy spy and he has a hold on Maida to find out more and more intelligence on wartime materiel and personnel movements."}], "Mongoose R.I.P.": [{"Spy novels about North America and the military": "'''''Mongoose R.I.P.''''' is a 1988 Blackford Oakes novel by William F. Buckley, Jr. It is the eighth of 11 novels in the series.\nCIA agent Blackford Oakes is sent to Cuba to determine the feasibility of overthrowing Fidel Castro, following the Cuban Missile Crisis in 1963."}], "The Betrayers": [{"Spy novels about North America and the military": "'''''The Betrayers''''' is the tenth novel in the Matt Helm spy series by Donald Hamilton, which originated with ''Death of a Citizen'' in 1960. This novel was first published in 1966. It was reissued in 2014 by Titan Books."}], "The Man with the Golden Gun (novel)": [{"Spy novels about North America and the military": "'''''The Man with the Golden Gun''''' is the twelfth and final novel in Ian Fleming's James Bond series and thirteenth Bond book overall. It was first published by Jonathan Cape in the UK on 1 April 1965, eight months after the author's death. The novel was not as detailed or polished as the others in the series, leading to poor but polite reviews. Despite that, the book was a best-seller."}], "Kolymsky Heights": [{"Spy novels about North America and the military": "'''''Kolymsky Heights''''' is a 1994 thriller novel by Lionel Davidson. It was his first thriller novel in 16 years, following ''The Chelsea Murders''.\n\n\nA coded message is smuggled out of Russia, a plea for help from the director of a super-secret laboratory deep in the frozen wastes of Siberia. The note is addressed to Johnny Porter, a Canadian Indian of the Gitxsan tribe with a genius for languages. The CIA recruits Porter, who infiltrates Russia, first posing as a Korean sailor on a tramp freighter, then as a Chukchee driver called Khodyan. Working at a transport company, he befriends an employee who gives him sufficient spare parts to build a bobik truck, which he assembles in a cave. Porter also befriends the local doctor, Tanya Komarova, who is also working for the CIA, and they become lovers. With her help, he infiltrates the research facility by switching places with an Evenk employee. The director, Ephraim Rogachev, reveals to Porter the research they have been conducting, including a cure for blindness which the Soviets are concealing because of the military applications of the technology."}], "A Tract of Time": [{"Spy novels about North America and the military": "'''''A Tract of Time''''' is an antiwar novel from 1966 by Smith Hempstone, that covers the time period about 1960, when there was an attempted coup of South Vietnamese President Ngo Dinh Diem. Even as the United States backed Diem's government during the war, its American advisers worked with the Montagnard people who opposed Diem, to help them fight the Viet Cong, whom they also opposed. The book follows one CIA operative, Harry Coltart, as he works with the Montagnard mountain tribesmen in the Central Highlands. Harry is initially successful in getting the Montagnards to fight against the Viet Cong, but then the Montagnards are betrayed and South Vietnamese troops are sent in. Harry has to be rescued as the Montagnards join the Viet Cong."}]}}} +{"query": "Prehistoric animals of Europe and Africa, that are not from the Miocene period", "docs": ["Columbian mammoth", "Theropithecus oswaldi", "Paracamelus", "Mammuthus meridionalis", "Mammuthus lamarmorai", "Metailurus", "Dinofelis", "Homotherium", "Agriotherium", "Pygmy mammoth", "Steppe mammoth", "Homo erectus", "Pachycrocuta", "Mammuthus rumanus", "Mammuthus creticus", "Necromantis", "Mammuthus subplanifrons"], "original_query": "Mammals of Europe that are also Prehistoric animals of Africa but not Miocene mammals", "scores": null, "metadata": {"template": "_ that are also _ but not _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "animals"}} +{"query": "Children's books published by Grosset & Dunlap in the 1960s", "docs": ["The Password to Larkspur Lane", "The Phantom of Pine Hill", "The Invisible Intruder", "The Clue of the Whistling Bagpipes", "The Bungalow Mystery", "The Clue of the Broken Locket"], "original_query": "1960s children's books that are also Grosset & Dunlap books but not The Hardy Boys books", "scores": null, "metadata": {"template": "_ that are also _ but not _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"The Password to Larkspur Lane": ["Definitely relevant"], "The Phantom of Pine Hill": ["Likely relevant"], "The Invisible Intruder": ["Definitely relevant"], "The Clue of the Whistling Bagpipes": ["Likely relevant"], "The Bungalow Mystery": ["Likely relevant"], "The Clue of the Broken Locket": ["Likely relevant"]}, "evidence_ratings": {"The Password to Larkspur Lane": ["Complete"], "The Phantom of Pine Hill": ["Partial"], "The Invisible Intruder": ["Partial"], "The Clue of the Whistling Bagpipes": ["Partial"], "The Bungalow Mystery": ["Partial"], "The Clue of the Broken Locket": ["Partial"]}, "attributions": {"The Password to Larkspur Lane": [{"Children's books published by Grosset & Dunlap": "Collectors speculate publisher Grosset & Dunlap commissioned an updated illustration of the same scene during the transition from Gillies to new series artist Rudy Nappi in 1953.", "in the 1960s": "This art later appeared on British dust jackets for this volume in 1960."}], "The Phantom of Pine Hill": [{"Children's books published by Grosset & Dunlap in the 1960s": "It was first published in 1965 under the pseudonym Carolyn Keene."}], "The Invisible Intruder": [{"Children's books published by Grosset & Dunlap in the 1960s": "'''''The Invisible Intruder''''' is the 46th volume in the Nancy Drew Mystery Stories series. It was first published in 1969 under Carolyn Keene. The actual author was ghostwriter Harriet Stratemeyer Adams.\nNancy and her friends are invited on a ghost-hunting tour, visiting various locations reputed to be haunted. They gather clues that point to a more mundane explanation."}], "The Clue of the Whistling Bagpipes": [{"Children's books published by Grosset & Dunlap in the 1960s": "It was first published in 1964 under the pseudonym Carolyn Keene."}], "The Bungalow Mystery": [{"in the 1960s": "It was the final volume edited by Edward Stratemeyer before his death. His daughter, Harriet Stratemeyer Adams, extensively revised the novel in 1960."}], "The Clue of the Broken Locket": [{"1960s": "It was later revised by Harriet Stratemeyer in 1965, and the story was mostly changed with a few elements of the original.", "Children's": "he Blairs, who are famous actors, decide to adopt the babies as a publicity stunt, as they hope to raise the children as actors, which will help their own careers."}]}}} +{"query": "World War II prisoner of war films that won a European Film Award", "docs": ["Life Is Beautiful", "Ida (film)"], "original_query": "European Film Awards winners (films) that are also World War II prisoner of war films", "scores": null, "metadata": {"template": "_ that are also _", "domain": "films", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Life Is Beautiful": ["Definitely relevant"], "Ida (film)": ["Definitely relevant"]}, "evidence_ratings": {"Life Is Beautiful": ["Complete"], "Ida (film)": ["Complete"]}, "attributions": {"Life Is Beautiful": [{"World War II prisoner of war films": "The film was partially inspired by the book ''In the End, I Beat Hitler'' by Rubino Romeo Salmon\u00ec and by Benigni's father, who spent two years in the Bergen-Belsen concentration camp during World War II.", "that won a European Film Award": "The movie won the Grand Prix at the 1998 Cannes Film Festival, nine David di Donatello Awards (including Best Film), five Nastro d'Argento Awards in Italy, two European Film Awards, and three Academy Awards, including Best Foreign Language Film and Best Actor for Benigni, the first for a male non-English performance."}], "Ida (film)": [{"World War II prisoner of war films": "Orphaned as an infant during the German occupation of World War II, she must meet her aunt, a former Communist state prosecutor and only surviving relative, who tells her that her parents were Jewish.", "that won a European Film Award": "It had earlier been selected as Best Film of 2014 by the European Film Academy"}]}}} +{"query": "what are Oceanian realm fauna that are also both Birds of North America and Fauna of Europe", "docs": ["Sooty tern", "Bulwer's petrel", "Black noddy", "Bar-tailed godwit", "Masked booby", "Red-footed booby", "Roseate tern"], "original_query": "Oceanian realm fauna that are also both Birds of North America and Fauna of Europe", "scores": null, "metadata": {"template": "_ that are also both _ and _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "animals"}} +{"query": "1970's harpercollins childrens books published posthumously.", "docs": ["The First Four Years (novel)", "The Father Christmas Letters"], "original_query": "1970s children's books that are also both Books published posthumously and HarperCollins books", "scores": null, "metadata": {"template": "_ that are also both _ and _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"The First Four Years (novel)": ["Definitely relevant"], "The Father Christmas Letters": ["Definitely relevant"]}, "evidence_ratings": {"The First Four Years (novel)": ["Complete"], "The Father Christmas Letters": ["Complete"]}, "attributions": {"The First Four Years (novel)": [{"1970's harpercollins childrens books": "The series had initially concluded at eight children's novels following Wilder to mature age and her marriage with Almanzo Wilder."}], "The Father Christmas Letters": [{"1970's harpercollins childrens books published posthumously.": "'''''The Father Christmas Letters''''', also known as '''''Letters from Father Christmas''''', are a collection of letters written and illustrated by J. R. R. Tolkien between 1920 and 1943 for his children, from Father Christmas. They were released posthumously by the Tolkien estate on 2 September 1976, the 3rd anniversary of Tolkien's death. They were edited by Baillie Tolkien, second wife of his youngest son, Christopher. The book was warmly received by critics, and it has been suggested that elements of the stories inspired parts of Tolkien's ''The Lord of the Rings''."}]}}} +{"query": "Cenzoic birds that are not found in New Zealand", "docs": ["Cuban kestrel", "Kaua\u02bbi palila", "Maui Nui finch", "Apteribis", "Scissor-billed koa finch", "Puerto Rican quail-dove", "Aepyornis", "Stilt-owl", "Xestospiza conica", "N\u0113n\u0113-nui", "High-billed crow", "Cuban pauraque", "Xestospiza", "Mullerornis", "Thambetochen", "Xestospiza fastigialis", "Mauritius starling", "Moa-nalo", "Vorombe", "Highland finch"], "original_query": "Endemic fauna that are also Cenozoic birds but not Fauna of New Zealand", "scores": null, "metadata": {"template": "_ that are also _ but not _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "animals"}} +{"query": "1947 Science Linguistics books", "docs": ["LTI \u2013 Lingua Tertii Imperii", "Meaning and Necessity"], "original_query": "Science books that are also both Linguistics books and 1947 books", "scores": null, "metadata": {"template": "_ that are also both _ and _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"LTI \u2013 Lingua Tertii Imperii": ["Likely relevant"], "Meaning and Necessity": ["Definitely relevant"]}, "evidence_ratings": {"LTI \u2013 Lingua Tertii Imperii": ["Partial"], "Meaning and Necessity": ["Complete"]}, "attributions": {"LTI \u2013 Lingua Tertii Imperii": [{"1947 Science Linguistics books": "'''''LTI \u2013 Lingua Tertii Imperii: Notizbuch eines Philologen''''' (1947) is a book by Victor Klemperer, Professor of Literature at the Dresden University of Technology. The title, half in Latin and half in German, translates to \"'''''The Language of the Third Reich: A Philologist's Notebook'''''\"; the book is published in English translation as '''''The Language of the Third Reich'''''.\n''Lingua Tertii Imperii'' studies the way that Nazi propaganda altered the German language to inculcate people with the ideas of Nazism. The book was written under the form of personal notes which Klemperer wrote in his diary, especially from the rise of the Nazi regime in 1933, and even more after 1935, when Klemperer, was stripped of his academic title because he was of Jewish descent. His diary became a notebook in which he noted and commented on the linguistic relativity of the German used by Nazi officials, ordinary citizens, and even fellow Jews. Klemperer wrote the book, based on his notes, in 1945\u20131946."}], "Meaning and Necessity": [{"947 Science Linguistics books": "'''''Meaning and Necessity: A Study in Semantics and Modal Logic''''' (1947; enlarged edition 1956) is a book about semantics and modal logic by the philosopher Rudolf Carnap. The book, in which Carnap discusses the nature of linguistic expressions, was a continuation of his previous work in semantics in ''Introduction to Semantics'' (1942) and ''Formalization of Logic'' (1943)."}]}}} +{"query": "1960s romance films that are also 1960s musical films excluding Musical comedy films", "docs": ["Guitars Sound Softly Through the Night", "Paris Blues", "Naila (film)", "The Umbrellas of Cherbourg", "Lakhon Mein Aik", "Meenda Sorgam", "Heaven Is Never a Great Distance", "Los Tarantos", "Roustabout (film)", "West Side Story (1961 film)", "Thillana Mohanambal", "All the Fine Young Cannibals"], "original_query": "1960s romance films that are also 1960s musical films but not Musical comedy films", "scores": null, "metadata": {"template": "_ that are also _ but not _", "domain": "films", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Guitars Sound Softly Through the Night": ["Definitely relevant"], "Paris Blues": ["Definitely relevant"], "Naila (film)": ["Definitely relevant"], "The Umbrellas of Cherbourg": ["Definitely relevant"], "Lakhon Mein Aik": ["Definitely relevant"], "Meenda Sorgam": ["Definitely relevant"], "Heaven Is Never a Great Distance": ["Likely relevant"], "Los Tarantos": ["Likely relevant"], "Roustabout (film)": ["Likely relevant"], "West Side Story (1961 film)": ["Definitely relevant"], "Thillana Mohanambal": ["Definitely relevant"], "All the Fine Young Cannibals": ["Likely relevant"]}, "evidence_ratings": {"Guitars Sound Softly Through the Night": ["Complete"], "Paris Blues": ["Complete"], "Naila (film)": ["Complete"], "The Umbrellas of Cherbourg": ["Partial"], "Lakhon Mein Aik": ["Complete"], "Meenda Sorgam": ["Complete"], "Heaven Is Never a Great Distance": ["Partial"], "Los Tarantos": ["Partial"], "Roustabout (film)": ["Partial"], "West Side Story (1961 film)": ["Complete"], "Thillana Mohanambal": ["Complete"], "All the Fine Young Cannibals": ["Partial"]}, "attributions": {"Guitars Sound Softly Through the Night": [{"1960s romance films that are also 1960s musical films excluding Musical comedy films": "'''''Guitars Sound Softly Through the Night''''' (German: '''''Gitarren klingen leise durch die Nacht''''') is a 1960 Austrian romantic musical film directed by Hans Deppe and starring Fred Bertelmann, Margit N\u00fcnke and Vivi Bach."}], "Paris Blues": [{"1960s romance films that are also 1960s musical films excluding Musical comedy films": "'''''Paris Blues''''' is a 1961 American musical romantic drama film"}], "Naila (film)": [{"1960s romance films that are also 1960s musical films excluding Musical comedy films": "'''''Naila''''' is a 1965 Pakistani musical romance drama film"}], "The Umbrellas of Cherbourg": [{"1960s romance films that are also 1960s musical films excluding Musical comedy films": "'''''The Umbrellas of Cherbourg''''' () is a 1964 musical romantic drama film written"}], "Lakhon Mein Aik": [{"1960s romance films that are also 1960s musical films excluding Musical comedy films": "'''''Lakhon Mein Aik''''' (; ) is a 1967 Pakistani romantic musical film directed by Raza Mir and written by Zia Sarhadi."}], "Meenda Sorgam": [{"1960s romance films that are also 1960s musical films excluding Musical comedy films": "'''''Meenda Sorgam''''' () is a 1960 Indian Tamil-language romantic musical film"}], "Heaven Is Never a Great Distance": [{"1960s romance films that are also 1960s musical films excluding Musical comedy films": "'''''Heaven Is Never a Great Distance''''' ''(Persian title: Behesht door nist- )'' is a 1969 Iranian ''Persian-genre'' dance Romantic film"}], "Los Tarantos": [{"1960s romance films that are also 1960s musical films": "'''''Los Tarantos''''' is a 1963 Spanish musical drama film", "romance": "The film is based on the play ''La historia de los Tarantos'' written by Alfredo Ma\u00f1as, and inspired by ''Romeo and Juliet'' by William Shakespeare.\nThe love between two gipsies, Juana La Zoronga and Rafael El Taranto, from different families in Barcelona is thwarted by the enmity between their respective parents. Rafael sees Juana dance at a gipsy wedding, and is captivated by her beauty and charm, and they fall in love, aided by their younger siblings who are secretly friends and sympathetic to the young lovers."}], "Roustabout (film)": [{"1960s romance films that are also 1960s": "'''''Roustabout''''' is a 1964 American musical feature film starring Elvis Presley as a singer who takes a job working with a struggling carnival. The film was produced by Hal Wallis and directed by John Rich from a screenplay by Anthony Lawrence and Allan Weiss. The screenplay was nominated for a Writers Guild of America award for best written American musical although ''Roustabout'' received a lukewarm review in ''Variety''. The film's soundtrack album was one of Elvis Presley's most successful, reaching no. 1 on the Billboard Album Chart."}], "West Side Story (1961 film)": [{"1960s romance films that are also 1960s musical films excluding Musical comedy films": "'''''West Side Story''''' is a 1961 American musical romantic drama film directed by Robert Wise and Jerome Robbins."}], "Thillana Mohanambal": [{"1960s musical films excluding Musical comedy films": "'''''Thillana Mohanambal''''' () is a 1968 Indian Tamil-language musical drama film written, directed and produced by A. P. Nagarajan.", "romance films": "It tells the story of Shanmugasundaram, a ''nadaswaram'' player who falls in love with Mohanambal, a ''Bharatanatyam'' dancer who reciprocates his feelings, but unfortunate circumstances and their egoistic nature prevents them from confessing their love for one another."}], "All the Fine Young Cannibals": [{"1960s": "'''''All the Fine Young Cannibals''''' is a 1960 American film directed by Michael Anderson, based on the novel by Rosamond Marshall, starring Robert Wagner, Natalie Wood, Susan Kohner, George Hamilton and Pearl Bailey."}]}}} +{"query": "Non-fiction history books from 2019 about genocide", "docs": ["The Thirty-Year Genocide", "The Volunteer (book)"], "original_query": "2019 non-fiction books that are also History books about genocide", "scores": null, "metadata": {"template": "_ that are also _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"The Thirty-Year Genocide": ["Definitely relevant"], "The Volunteer (book)": ["Definitely relevant"]}, "evidence_ratings": {"The Thirty-Year Genocide": ["Complete"], "The Volunteer (book)": ["Partial"]}, "attributions": {"The Thirty-Year Genocide": [{"Non-fiction history books from 2019 about genocide": "'''The Thirty-Year Genocide: Turkey's Destruction of Its Christian Minorities, 1894\u20131924''''' is a 2019 history book written by Benny Morris and Dror Ze'evi. They argue that the Armenian genocide and other contemporaneous persecution of Christians in the Ottoman Empire constitute an extermination campaign, or genocide, carried out by the Ottoman Empire against its Christian subjects."}], "The Volunteer (book)": [{"Non-fiction history books from 2019": "'''''The Volunteer: The True Story of the Resistance Hero Who Infiltrated Auschwitz''''' (British title; the American edition is titled '''''The Volunteer: One Man's Mission to Lead an Underground Army Inside Auschwitz and Stop the Holocaust''''') is a 2019 book which presents research by British writer Jack Fairweather, a former ''Washington Post'' war correspondent, into the life of Witold Pilecki, a Polish soldier and Home Army resistance fighter who infiltrated the infamous Auschwitz concentration camp."}]}}} +{"query": "cultural geography and Science books but not about creativity", "docs": ["The China\u2013Pakistan Axis", "Clash of Civilizations", "The Nine Nations of North America", "When Work Disappears", "Jewish Roots in Ukraine and Moldova", "Hellenic Nomarchy", "The End of History and the Last Man", "Searching for Whitopia", "Where Once We Walked", "The Journey of Man", "International Encyclopedia of Human Geography", "The Real Eve", "Geographia Neoteriki", "The Wealth and Poverty of Nations", "Jewish Roots in Poland"], "original_query": "Science books that are also Books about cultural geography but not Books about creativity", "scores": null, "metadata": {"template": "_ that are also _ but not _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"The China\u2013Pakistan Axis": ["Definitely relevant"], "Clash of Civilizations": ["Likely relevant"], "The Nine Nations of North America": ["Likely relevant"], "When Work Disappears": ["Likely relevant"], "Jewish Roots in Ukraine and Moldova": ["Definitely relevant"], "Hellenic Nomarchy": ["Likely relevant"], "The End of History and the Last Man": ["Definitely relevant"], "Searching for Whitopia": ["Likely relevant"], "Where Once We Walked": ["Likely relevant"], "The Journey of Man": ["Definitely relevant"], "International Encyclopedia of Human Geography": ["Definitely relevant"], "The Real Eve": ["Definitely relevant"], "Geographia Neoteriki": ["Definitely relevant"], "The Wealth and Poverty of Nations": ["Definitely relevant"], "Jewish Roots in Poland": ["Likely relevant"]}, "evidence_ratings": {"The China\u2013Pakistan Axis": ["Complete"], "Clash of Civilizations": ["Complete"], "The Nine Nations of North America": ["Partial"], "When Work Disappears": ["Complete"], "Jewish Roots in Ukraine and Moldova": ["Partial"], "Hellenic Nomarchy": ["Partial"], "The End of History and the Last Man": ["Complete"], "Searching for Whitopia": ["Partial"], "Where Once We Walked": ["Partial"], "The Journey of Man": ["Complete"], "International Encyclopedia of Human Geography": ["Complete"], "The Real Eve": ["Complete"], "Geographia Neoteriki": ["Complete"], "The Wealth and Poverty of Nations": ["Complete"], "Jewish Roots in Poland": ["Partial"]}, "attributions": {"The China\u2013Pakistan Axis": [{"cultural geography": "'''''The China\u2013Pakistan Axis: Asia's New Geopolitics''''' is a book by the British author Andrew Small exploring the cultural and political ties as well as depicting the relationship between Pakistan and China, released in early 2015."}], "Clash of Civilizations": [{"cultural geography and Science books": "The '''Clash of Civilizations''' is a thesis that people's cultural and religious identities will be the primary source of conflict in the post\u2013Cold War world. The American political scientist Samuel P. Huntington argued that future wars would be fought not between countries, but between cultures."}], "The Nine Nations of North America": [{"cultural geography": "'''''The Nine Nations of North America''''' is a 1981 book by Joel Garreau, in which the author suggests that North America can be divided into nine nations, which have distinctive economic and cultural features."}], "When Work Disappears": [{"cultural geography": "Wilson's argument is that the disappearance of work and the consequences of that disappearance for both social and cultural life are the central problems in the inner-city ghetto.", "Science books": "Wilson's book uses evidence from large-scale scientific surveys in the ghetto and information culled from ethnographic interviews of ghetto residents in order to create a complete picture of the problems that face the residents."}], "Jewish Roots in Ukraine and Moldova": [{"cultural geography and Science books but not about creativity": "'Jewish Roots in Ukraine and Moldova''''' (full title: ''Jewish Roots in Ukraine and Moldova: Pages from the Past and Archival Inventories'') is a book created by genealogist Miriam Weiner and co-published by The Miriam Weiner Routes to Roots Foundation and YIVO Institute for Jewish Research. A searchable database of updated archival holdings listed in the book"}], "Hellenic Nomarchy": [{"cultural geography": "It advocated the ideals of freedom, social justice and equality as the main principles of a well-governed society, making it the most important theoretical monument of Greek republicanism."}], "The End of History and the Last Man": [{"cultural geography and Science books but not about creativity": "who define human history as a linear progression, from one socioeconomic epoch to another."}], "Searching for Whitopia": [{"cultural geography": "African American journalist Rich Benjamin documents his journeys to find out why more and more white Americans move to small towns and areas that are, for the most part, white, and to explain why Whitopias are growing and what it means for the United States."}], "Where Once We Walked": [{"cultural geography and Science books but not about creativity": "'''''Where Once We Walked''''' (full title: ''Where Once We Walked: A Guide to the Jewish Communities Destroyed in The Holocaust''), compiled by noted genealogist Gary Mokotoff and Sallyann Amdur Sack with Alexander Sharon, is a gazetteer of 37,000 town names in Central and Eastern Europe focusing on those with Jewish populations in the 19th and first half of the 20th centuries and most of whose Jewish communities were almost or completely destroyed during The Holocaust."}], "The Journey of Man": [{"cultural geography and Science books but not about creativity": "American geneticist and anthropologist, in which he uses techniques and theories of genetics and evolutionary biology to trace the geographical dispersal of early human migrations out of Africa"}], "International Encyclopedia of Human Geography": [{"cultural geography and Science books": "The '''''International Encyclopedia of Human Geography''''' is a 2009 academic reference work covering human geography."}], "The Real Eve": [{"cultural geography and Science books": "'''''The Real Eve: Modern Man's Journey Out of Africa''''' is a popular science book about the evolution of modern humans written by British geneticist Stephen Oppenheimer."}], "Geographia Neoteriki": [{"cultural geography and Science books but not about creativity": "'''''Geographia Neoteriki''''' ( ''Modern Geography'') is a geography book written in Greek by Daniel Philippidis and Grigorios Konstantas and printed in Vienna in 1791. It focused on both the physical and human geography features of the European continent and especially on Southeastern Europe, and is considered one of the most remarkable works of the modern Greek Enlightenment."}], "The Wealth and Poverty of Nations": [{"cultural geography and Science books": "Landes attempted to explain why some countries and regions experienced near miraculous periods of explosive growth while the rest of the world stagnated. The book compared the long-term economic histories of different regions, specifically Europe, United States, Japan, China, the Arab world, and Latin America. In addition to analyzing economic and cliometric figures, he credited intangible assets, such as culture and enterprise, to explain economic success or failure.", "but not about creativity": "In addition to analyzing economic and cliometric figures, he credited intangible assets, such as culture and enterprise, to explain economic success or failure."}], "Jewish Roots in Poland": [{"cultural geography": "The book includes archival holdings of the Polish State Archives, the Jewish Historical Institute in Warsaw, local town hall documents throughout Poland, Holocaust documents found in the archives of the death camps located in Auschwitz near Krakow and Majdanek near Lublin. The book also features document examples, maps, antique postcards depicting towns and daily life, and modern-day photographs."}]}}} +{"query": "2010s non-fiction books that are about social psychology but not crowd psychology", "docs": ["The Boy in the River", "Silent No More (book)", "A Very English Scandal", "Simple Choices", "A History of Marriage", "Quiet Power", "The End of American Childhood", "IGen (book)", "Scarcity: Why Having Too Little Means So Much", "The Argonauts", "The Righteous Mind", "Battle Hymn of the Tiger Mother", "The New Childhood", "Far from the Tree", "The Importance of Being Little", "The Coddling of the American Mind"], "original_query": "2010s non-fiction books that are also Books about social psychology but not Books about crowd psychology", "scores": null, "metadata": {"template": "_ that are also _ but not _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"The Boy in the River": ["Likely relevant"], "Silent No More (book)": ["Definitely relevant"], "A Very English Scandal": ["Likely relevant"], "Simple Choices": ["Definitely relevant"], "A History of Marriage": ["Likely relevant"], "Quiet Power": ["Likely relevant"], "The End of American Childhood": ["Likely relevant"], "IGen (book)": ["Definitely relevant", "Likely not relevant"], "Scarcity: Why Having Too Little Means So Much": ["Definitely relevant"], "The Argonauts": ["Likely relevant"], "The Righteous Mind": ["Definitely relevant"], "Battle Hymn of the Tiger Mother": ["Likely relevant"], "The New Childhood": ["Likely relevant"], "Far from the Tree": ["Definitely relevant"], "The Importance of Being Little": ["Likely relevant"], "The Coddling of the American Mind": ["Likely relevant"]}, "evidence_ratings": {"The Boy in the River": ["Partial"], "Silent No More (book)": ["Partial"], "A Very English Scandal": ["Partial"], "Simple Choices": ["Complete"], "A History of Marriage": ["Partial"], "Quiet Power": ["Partial"], "The End of American Childhood": ["Partial"], "IGen (book)": ["Complete", "Complete"], "Scarcity: Why Having Too Little Means So Much": ["Complete"], "The Argonauts": ["Partial"], "The Righteous Mind": ["Complete"], "Battle Hymn of the Tiger Mother": ["Partial"], "The New Childhood": ["Partial"], "Far from the Tree": ["Complete"], "The Importance of Being Little": ["Partial"], "The Coddling of the American Mind": ["Partial"]}, "attributions": {"The Boy in the River": [{"2010s non-fiction books that are about social psychology": "'''''The Boy in the River''''' is a 2012 book by the British author and criminologist Richard Hoskins about the 'Adam' murder case."}], "Silent No More (book)": [{"2010s non-fiction books that are about social psychology but not crowd psychology": "'''''Silent No More: Victim 1's Fight for Justice Against Jerry Sandusky''''' is a 2012 book by Aaron Fisher, identified as \"Victim 1\" in the Penn State child sex abuse scandal. Fisher is called \"Victim 1\" because it was his reporting his abuse to high school officials that set off the investigation that led to Sandusky's conviction. The book follows Fisher's experience from the beginning of his interaction with Jerry Sandusky at The Second Mile through Sandusky's conviction of 45 of 48 counts related to child sex abuse."}], "A Very English Scandal": [{"non-fiction books": "'''''A Very English Scandal''''' is a true crime non-fiction novel by John Preston.", "2010s": "It was first published on 5 May 2016 by Viking Press and by Other Press in the United States."}], "Simple Choices": [{"2010s non-fiction books that are about social psychology but not crowd psychology": "'''''Simple Choices: Thoughts on choosing environments that support who your child is meant to be''''' is a parenting book by Lisa Graham Keegan published in 2013.\n\n\n''Simple Choices'' begins with Keegan's memoir of building a family through divorce, abandonment, adoption, mental impairment, sexual identity and all the challenges life has to offer."}], "A History of Marriage": [{"2010s non-fiction books that are about social psychology but not crowd psychology": "'''''A History of Marriage''''', published by Seven Stories Press, is a non-fiction book by Elizabeth Abbott, the Canadian author of ''A History of Celibacy'' (1999) and ''A History of Mistresses'' (2003) that combines general history and personal histories of marriage. The book is a study of mostly North American rituals of courting, nuptials, marriage, sex, child-raising and divorce. Some topics covered are relative ages at which various societies from Chinese to Mormon married off their girls; details of the satisfying marriage of Martin Luther and former nun Katharina von Bora; the ruptured family units of Native American children removed to residential schools; the popularity of so-called Boston marriages (depicted by Henry James in ''The Bostonians'') between like-minded women who resisted conventional marriage but weren't necessarily lesbian; and the scarcity of sponges used for contraception by Northern women during the Civil War because of the cut-off in supply from Florida. ''A History of Marriage'' was a finalist for the 2010 Governor General's Literary Award for non-fiction.\n\n\nA paperback edition was published in August 2015 by Seven Stories Press."}], "Quiet Power": [{"2010s non-fiction books that are about social psychology but not crowd psychology": "'''''Quiet Power: The Secret Strengths of Introverts''''' is a 2016 non-fiction book written by Susan Cain with Gregory Mone and Erica Moroz, and illustrated by Grant Snider.", "social psychology but": "''Quiet Power'' discusses the distinction between introversion and shyness; deeper student engagement versus conventional expectations of class participation; speaking in front of groups; individual versus group work; introvert-friendly methods of structuring group work; and use of social media in education. The book focuses on shyness in addition to introversion, saying that shyness involves fear of social judgment."}], "The End of American Childhood": [{"2010s non-fiction books that are about social psychology": "'''''The End of American Childhood: A History of Parenting from Life on the Frontier to the Managed Child''''' was written by historian Paula S. Fass and published by Princeton University Press in 2016."}], "IGen (book)": [{"2010s non-fiction books that are about social psychology but not crowd psychology": "'''''iGen: Why Today's Super-Connected Kids Are Growing Up Less Rebellious, More Tolerant, Less Happy\u2014and Completely Unprepared for Adulthood\u2014and What That Means for the Rest of Us''''' is a 2017 nonfiction book by Jean Twenge which studies the lifestyles, habits and values of Americans born 1995\u20132012, the first generation to reach adolescence after smartphones became widespread."}, null], "Scarcity: Why Having Too Little Means So Much": [{"2010s non-fiction books that are about social psychology but not crowd psychology": "'''''Scarcity: Why Having Too Little Means So Much''''' is a 2013 book by behavioural economist Sendhil Mullainathan and psychologist Eldar Shafir. The authors discuss the role of scarcity in creating, perpetuating, and alleviating poverty. The book also proposes several ideas for how individuals and groups of people can handle scarcity to achieve success and satisfaction."}], "The Argonauts": [{"2010s": "'''''The Argonauts''''' is a book by poet and critic Maggie Nelson, published in 2015", "social psychology": "The book discusses her romantic relationship with the transgender artist Harry Dodge leading to her pregnancy as well as topics ranging from the death of a parent, transgender embodiment, academia, familial relationships, and the limitations of language."}], "The Righteous Mind": [{"2010s non-fiction books that are about social psychology but not crowd psychology": "A simple graphic depicting survey data from the United States intended to support moral foundations theory.\n'''''The Righteous Mind: Why Good People are Divided by Politics and Religion''''' is a 2012 social psychology book by Jonathan Haidt, in which the author describes human morality as it relates to politics and religion.\nIn the first part of the book, the author uses research to demonstrate social intuitionism, how people's beliefs come primarily from their intuitions, and rational thought often comes after to justify initial beliefs. He cites David Hume and E. O. Wilson as thinkers who gave reason a relatively low estimation, as opposed to more popular thinkers who give reason a central place in moral cognition, such as Lawrence Kohlberg and his stages of moral reasoning."}], "Battle Hymn of the Tiger Mother": [{"2010s non-fiction books that are about social psychology": "'''''Battle Hymn of the Tiger Mother''''' is a book by American author and law professor Amy Chua that was published in 2011."}], "The New Childhood": [{"2010s non-fiction books that are about social psychology": "in December 2018.\nShapiro, who teaches at Temple University in the College of Liberal Arts, became interested in childhood development and digital play following his divorce, after playing video games with his two sons. He argues against strict screen time limits for kids, suggesting instead that parents should teach their kids how to use digital devices with integrity."}], "Far from the Tree": [{"2010s non-fiction books that are about social psychology": "'''''Far From the Tree: Parents, Children, and the Search for Identity''''' is a non-fiction book by Andrew Solomon published in November 2012 in the United States and two months later in the UK (under the title, ''Far from the Tree: A Dozen Kinds of Love''), about how families accommodate children with physical, mental and social disabilities and differences."}], "The Importance of Being Little": [{"2010s non-fiction books that are about social psychology but not crowd psychology": "'''''The Importance of Being Little: What Preschoolers Really Need from Grownups''''' is a 2016 book written by early childhood educator Erika Christakis that argues the importance of separating childhood from adulthood. It was published by Viking Press."}], "The Coddling of the American Mind": [{"2010s": "2018 book"}]}}} +{"query": "American novels about New Mexico from the 1980s", "docs": ["Talking God", "The Dark Wind", "A Thief of Time", "People of Darkness", "Contact (novel)", "The Cardinal of the Kremlin", "Skinwalkers (novel)", "Tiger Eyes"], "original_query": "1980s books that are also both American novels and Books about New Mexico", "scores": null, "metadata": {"template": "_ that are also both _ and _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Talking God": ["Likely relevant"], "The Dark Wind": ["Definitely relevant"], "A Thief of Time": ["Definitely relevant"], "People of Darkness": ["Definitely relevant"], "Contact (novel)": ["Definitely relevant"], "The Cardinal of the Kremlin": ["Likely relevant"], "Skinwalkers (novel)": ["Definitely relevant"], "Tiger Eyes": ["Definitely relevant"]}, "evidence_ratings": {"Talking God": ["Partial"], "The Dark Wind": ["Complete"], "A Thief of Time": ["Complete"], "People of Darkness": ["Partial"], "Contact (novel)": ["Complete"], "The Cardinal of the Kremlin": ["Partial"], "Skinwalkers (novel)": ["Partial"], "Tiger Eyes": ["Complete"]}, "attributions": {"Talking God": [{"American novels about New Mexico from the 1980s": "An unidentified man is found dead along the railroad tracks near Gallup, New Mexico."}], "The Dark Wind": [{"American novels about New Mexico from the 1980s": "'''The Dark Wind''''' is the fifth crime fiction novel in the Joe Leaphorn / Jim Chee Navajo Tribal Police series by Tony Hillerman, published in 1982. It is the second of the novels to feature Officer Jim Chee."}], "A Thief of Time": [{"from the 1980s": "''''A Thief of Time''''' is the eighth crime fiction novel Joe Leaphorn / Jim Chee Navajo Tribal Police series by Tony Hillerman, first published in 1988.", "about New Mexico": "#Albuquerque, NM\n\n#Aztec, NM\n#Bisti (Badlands and Trading Post), NM\n\n#Bitani Tsosi Wash, NM\n\n#Blanco, NM", "American novels": "'''''A Thief of Time''''' is the eighth crime fiction novel Joe Leaphorn / Jim Chee Navajo Tribal Police series by Tony Hillerman, first published in 1988."}], "People of Darkness": [{"American novels about New Mexico from the 1980s": "'''''People of Darkness''''' is the fourth crime fiction novel in the Joe Leaphorn / Jim Chee Navajo Tribal Police series by author Tony Hillerman, first published in 1980."}], "Contact (novel)": [{"American novels": "'''''Contact''''' is a 1985 hard science fiction novel by American scientist Carl Sagan.", "from the 1980s": "'''''Contact''''' is a 1985 hard science fiction novel by American scientist Carl Sagan.", "about New Mexico": "She eventually becomes the director of \"Project Argus\", a radiotelescope array in New Mexico dedicated to the search for extraterrestrial intelligence (SETI)."}], "The Cardinal of the Kremlin": [{"about New Mexico": "The mole, a lesbian named Dr. Beatrice Taussig who unluckily falls in love with Gregory's fianc\u00e9e, eventually gives up Bisyarina to the FBI out of guilt, and the Hostage Rescue Team later saves Gregory from his Soviet captors in a shabby desert safe house in New Mexico.", "from the 1980s": "'''''The Cardinal of the Kremlin''''' is an espionage thriller novel, written by Tom Clancy and released on May 20, 1988.", "American": "Ryan, who is part of the American arms negotiation team, travels to Moscow for the arms reduction talks."}], "Skinwalkers (novel)": [{"American novels about New Mexico from the 1980s": "'''''Skinwalkers''''' is the seventh crime-fiction novel in the Joe Leaphorn/Jim Chee Navajo Tribal Police series by author Tony Hillerman published in 1986. The film version,''Skinwalkers'', was adapted for television for the PBS ''Mystery!'' series in 2002."}], "Tiger Eyes": [{"American novels about New Mexico from the 1980s": "'''''Tiger Eyes''''' is a young adult novel written by Judy Blume in 1981 about a 15-year-old girl attempting to cope with the unexpected death of her father. In 2012, the novel was adapted into a film of the same name, directed by Judy's son, Lawrence Blume, and starring Willa Holland as Davey Wexler. Davey is often sad, depressed and defensive on everything. She has a short breakdown at her brother, Jason. Her mom becomes incapable of doing things about the house, making Davey more depressed.\nDavis \u201cDavey\u201d Wexler, along with her mother, Gwen, and her little brother, Jason, have just attended the funeral of her father, Adam, who was shot to death in a holdup at their 7-Eleven convenience store in Atlantic City. After lying in bed for days on end and not eating, Davey starts her tenth year of school, but faints\non her first day from anxiety. She goes for a checkup, and the doctor explains Davey is having panic attacks.\n\n\nDavey's mother, Gwen, decides they need to get away for awhile and takes up an offer from Adam's older sister, Bitsy, and his brother-in-law Walter to come stay with them in Los Alamos, New Mexico. A few days before they are scheduled to return to Atlantic City, Gwen receives news their store has been further vandalized, and she decides they're going to stay in Los Alamos through the end of the school year."}]}}} +{"query": "Fauna of Tibet and Arthropods of Asia", "docs": ["Spilosoma erythrozona", "Spilarctia leopardina", "Scorpiops langxian", "Areas galactina", "Bhutanitis thaidina", "Scorpiops tibetanus", "Mangina argus", "Callindra equitalis", "Sinopieris dubernardi", "Callindra principalis", "Eupithecia magica", "Ypthima sakra", "Athyma opalina", "Spilarctia nydia", "Rhyzodiastes orestes"], "original_query": "Arthropods of Asia that are also Fauna of Tibet", "scores": null, "metadata": {"template": "_ that are also _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "animals"}} +{"query": "Endemic fauna of Japan and Invertebrates of Asia but not Lepidoptera of Asia", "docs": ["Nebria pusilla", "Galloisiana nipponensis", "Agelena babai", "Galloisiana yuasai", "Galloisiana notabilis", "Clinidium veneficum", "Asemonea tanikawai", "Enteroplax yaeyamensis", "Sinopoda okinawana", "Galloisiana kiyosawai", "Calopteryx cornelia", "Parafontaria", "Galloisiana yezoensis", "Camponotus bishamon", "Cambaroides japonicus", "Boninena callistoderma", "Ogasawarana yoshiwarana", "Hirasea acutissima"], "original_query": "Invertebrates of Asia that are also Endemic fauna of Japan but not Lepidoptera of Asia", "scores": null, "metadata": {"template": "_ that are also _ but not _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "animals"}} +{"query": "Birds found in Sierra Madre Oriental that aren't found in Sierra Madre Occidental", "docs": ["Amethyst-throated mountaingem", "Red-lored amazon", "Gartered trogon", "Spot-crowned woodcreeper", "Northern bobwhite", "Great curassow", "Maroon-fronted parrot", "Curve-winged sabrewing", "Crimson-collared grosbeak", "Greenish elaenia", "White-faced quail-dove", "Black thrush", "Singing quail", "Crested guan", "Blue-capped motmot", "Bearded wood partridge", "Golden-fronted woodpecker", "Tamaulipas pygmy owl"], "original_query": "Birds of the Sierra Madre Oriental that are not Birds of the Sierra Madre Occidental", "scores": null, "metadata": {"template": "_ that are not _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "animals"}} +{"query": "1992 non-American independent films", "docs": ["Fortress (1992 film)", "Man Bites Dog (film)", "The Colours of My Father: A Portrait of Sam Borenstein", "The Bridge (1992 film)", "The Little Punker", "A Place in the World (film)", "The Crying Game", "Romper Stomper", "Star Time (film)", "The Last Days of Chez Nous", "Swoon (film)"], "original_query": "1992 independent films that are not American independent films", "scores": null, "metadata": {"template": "_ that are not _", "domain": "films", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Fortress (1992 film)": ["Likely relevant"], "Man Bites Dog (film)": ["Definitely relevant"], "The Colours of My Father: A Portrait of Sam Borenstein": ["Likely relevant"], "The Bridge (1992 film)": ["Likely relevant"], "The Little Punker": ["Definitely relevant"], "A Place in the World (film)": ["Likely relevant"], "The Crying Game": ["Likely relevant"], "Romper Stomper": ["Likely relevant"], "Star Time (film)": ["Definitely relevant"], "The Last Days of Chez Nous": ["Likely relevant"], "Swoon (film)": ["Likely relevant"]}, "evidence_ratings": {"Fortress (1992 film)": ["Partial"], "Man Bites Dog (film)": ["Complete"], "The Colours of My Father: A Portrait of Sam Borenstein": ["Partial"], "The Bridge (1992 film)": ["Complete"], "The Little Punker": ["Complete"], "A Place in the World (film)": ["Complete"], "The Crying Game": ["Partial"], "Romper Stomper": ["Partial"], "Star Time (film)": ["No evidence"], "The Last Days of Chez Nous": ["Partial"], "Swoon (film)": ["Partial"]}, "attributions": {"Fortress (1992 film)": [{"1992 non-American independent films": "'''''Fortress''''' is a 1992 science fiction action film"}], "Man Bites Dog (film)": [{"independent films": "''Man Bites Dog'' is shot in black and white on 16mm film and was produced on a shoe-string budget by four student filmmakers, led by director R\u00e9my Belvaux.", "1992 non-American": "'''''Man Bites Dog''''' (, literally \"It Happened Near Your Home\") is a 1992 Belgian black comedy crime mockumentary written, produced and directed by R\u00e9my Belvaux, Andr\u00e9 Bonzel and Beno\u00eet Poelvoorde, who are also the film's co-editor, cinematographer and lead actor respectively."}], "The Colours of My Father: A Portrait of Sam Borenstein": [{"1992 non-American": "'''''The Colours of My Father: A Portrait of Sam Borenstein''''' is a 1992 Canadian short animated documentary film directed by Joyce Borenstein."}], "The Bridge (1992 film)": [{"1992 non-American independent films": "a 1992 independent film", "non-American": "Directed by Sydney Macartney, it stars Saskia Reeves"}], "The Little Punker": [{"1992 non-American independent films": "a 1992 German animated feature independent dramedy film"}], "A Place in the World (film)": [{"1992 non-American independent films": "a 1992 Argentine drama film co-written, co-produced and directed by Adolfo Aristarain, and starring Federico Luppi."}], "The Crying Game": [{"1992 non-American": "'''''The Crying Game''''' is a 1992 thriller film written and directed by Neil Jordan, produced by Stephen Woolley, and starring Stephen Rea, Miranda Richardson, Jaye Davidson, Adrian Dunbar, Ralph Brown, and Forest Whitaker. The film explores themes of race, sex, nationality, and sexuality against the backdrop of the Troubles in Northern Ireland."}], "Romper Stomper": [{"1992 non-American independent films": "'''''Romper Stomper''''' is a 1992 Australian drama film"}], "Star Time (film)": [null], "The Last Days of Chez Nous": [{"1992 non-American": "'''''The Last Days of Chez Nous''''' is a 1992 Australian drama film directed by Gillian Armstrong and written by Helen Garner."}], "Swoon (film)": [{"1992 non-American independent films": "'''''Swoon''''' is a 1992 independent film written and directed by Tom Kalin."}]}}} +{"query": "Non Horror demon novels.", "docs": ["List of the Lost", "Blood Price", "The Black Spider", "The Castle in the Forest", "The Devil in Love (novel)", "Melmoth the Wanderer", "Practical Demonkeeping", "Artemis Fowl and the Lost Colony", "The Black Tattoo", "Good Omens", "Eric (novel)"], "original_query": "Demon novels that are not Horror novel series", "scores": null, "metadata": {"template": "_ that are not _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"List of the Lost": ["Definitely relevant"], "Blood Price": ["Definitely relevant"], "The Black Spider": ["Likely relevant"], "The Castle in the Forest": ["Likely relevant"], "The Devil in Love (novel)": ["Definitely relevant"], "Melmoth the Wanderer": ["Definitely relevant"], "Practical Demonkeeping": ["Likely relevant"], "Artemis Fowl and the Lost Colony": ["Definitely relevant"], "The Black Tattoo": ["Definitely relevant"], "Good Omens": ["Likely relevant"], "Eric (novel)": ["Definitely relevant"]}, "evidence_ratings": {"List of the Lost": ["Complete"], "Blood Price": ["Complete"], "The Black Spider": ["Partial"], "The Castle in the Forest": ["Partial"], "The Devil in Love (novel)": ["Complete"], "Melmoth the Wanderer": ["Complete"], "Practical Demonkeeping": ["Partial"], "Artemis Fowl and the Lost Colony": ["Complete"], "The Black Tattoo": ["Complete"], "Good Omens": ["Partial"], "Eric (novel)": ["Complete"]}, "attributions": {"List of the Lost": [{"Non Horror demon novels.": "The book is about a 1970s relay team in Boston who accidentally kill a homeless person, whose death brings misfortune to the team."}], "Blood Price": [{"Non Horror demon novels.": "He tells her that the killer is a demon, that she actually did see him disappear."}], "The Black Spider": [{"demon": "The hunter used his demonic powers to instill a curse in the kiss, which would ensure his payment."}], "The Castle in the Forest": [{"Non Horror demon novels.": "'''''The Castle in the Forest''''' is the last novel by writer Norman Mailer, published in the year of his death, 2007. It is the story of Adolf Hitler's childhood as seen through the eyes of Dieter, a demon sent to put him on his destructive path. The novel explores the idea that Hitler was the product of incest. It forms a thematic contrast with the writer's immediately previous novel ''The Gospel According to the Son'' (1999), which deals with the early life of Jesus. It received a good deal of praise, including a glowing review from Lee Siegel of ''The New York Times Book Review'', and was the ''New York Times'' Bestseller for 2007."}], "The Devil in Love (novel)": [{"Non Horror demon novels.": "Author of ''The Devil in Love, Jacques Cazotte''\n'''''The Devil in Love''''' (, 1772) is an occult romance by Jacques Cazotte which tells of a demon, or devil, who falls in love with a young Spanish nobleman named Don Alvaro, an amateur human dabbler, and attempts, in the guise of a young woman, to win his affections."}], "Melmoth the Wanderer": [{"Non Horror demon novels.": "'''''Melmoth the Wanderer''''' is an 1820 Gothic novel by Irish playwright, novelist and clergyman Charles Maturin. The novel's titular character is a scholar who sold his soul to the devil in exchange for 150 extra years of life, and searches the world for someone who will take over the pact for him, in a manner reminiscent of the Wandering Jew."}], "Practical Demonkeeping": [{"Non Horror demon novels": "His first novel, it deals with a demon from Hell and his master."}], "Artemis Fowl and the Lost Colony": [{"Non Horror demon novels.": "In Barcelona, Spain, Artemis Fowl II and Butler, his bodyguard, wait for a demon. They suddenly encounter a demon who transports Artemis through time."}], "The Black Tattoo": [{"Non Horror demon novels.": "'''''The Black Tattoo''''' is a young adult fantasy novel by Sam Enthoven, published in 2006. It deals with a boy, Charlie, becoming possessed by a demon that manifests itself in the form of a black tattoo on his body."}], "Good Omens": [{"demon novels.": "There are attempts by the angel Aziraphale and the demon Crowley to sabotage the coming of the end times, having grown accustomed to their comfortable surroundings in England."}], "Eric (novel)": [{"demon novels.": "the Demon King"}]}}} +{"query": "what are some Novels about legendary creatures and are also Paranormal romance, but not Contemporary fantasy.", "docs": ["Dark Possession", "Dark Prince", "Dark Fire (Feehan novel)", "Passion (novel)", "Dark Celebration", "Undead and Unwed", "Dark Destiny", "The Silver Kiss", "Undead and Unappreciated", "Dragon's Oath", "Dark Desire", "Dark Curse", "Dark Magic (novel)", "Dark Guardian (novel)", "Dark Secret (novel)", "Dark Challenge"], "original_query": "Novels about legendary creatures that are also Paranormal romance novels but not Contemporary fantasy novels", "scores": null, "metadata": {"template": "_ that are also _ but not _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Dark Possession": ["Likely relevant"], "Dark Prince": ["Definitely relevant"], "Dark Fire (Feehan novel)": ["Definitely relevant"], "Passion (novel)": ["Definitely relevant"], "Dark Celebration": ["Definitely relevant"], "Undead and Unwed": ["Definitely relevant"], "Dark Destiny": ["Definitely relevant"], "The Silver Kiss": ["Definitely relevant"], "Undead and Unappreciated": ["Definitely relevant"], "Dragon's Oath": ["Likely relevant"], "Dark Desire": ["Likely relevant"], "Dark Curse": ["Likely relevant"], "Dark Magic (novel)": ["Definitely relevant"], "Dark Guardian (novel)": ["Likely relevant"], "Dark Secret (novel)": ["Definitely relevant"], "Dark Challenge": ["Definitely relevant"]}, "evidence_ratings": {"Dark Possession": ["Partial"], "Dark Prince": ["Complete"], "Dark Fire (Feehan novel)": ["Complete"], "Passion (novel)": ["Complete"], "Dark Celebration": ["Partial"], "Undead and Unwed": ["Complete"], "Dark Destiny": ["Complete"], "The Silver Kiss": ["Complete"], "Undead and Unappreciated": ["Complete"], "Dragon's Oath": ["Partial"], "Dark Desire": ["Partial"], "Dark Curse": ["Complete"], "Dark Magic (novel)": ["Complete"], "Dark Guardian (novel)": ["Partial"], "Dark Secret (novel)": ["Complete"], "Dark Challenge": ["Partial"]}, "attributions": {"Dark Possession": [{"Paranormal": "'''''Dark Possession''''' is a paranormal/suspense novel written by American author Christine Feehan.", "but not Contemporary fantasy.": "'''''Dark Possession''''' is a paranormal/suspense novel written by American author Christine Feehan.", "Novels about legendary creatures": "Mary Ann views herself as just a human, but there is more to her than she realizes. She is able to go head to head with Manolito, who has no memory of creating the blood bond."}], "Dark Prince": [{"Paranormal romance,": "'''''Dark Prince''''' is a paranormal romance written by American author Christine Feehan.", "Novels about legendary creatures": "''Dark Prince'' introduces the Carpathians, a powerful and ancient race. They have many gifts, including the ability to shape-shift, and extended life spans, living well over many years. Though they feed on human blood, they don't kill their human prey, and for the most part live among them without detection.", "but not Contemporary fantasy.": "'''''Dark Prince''''' is a paranormal romance written by American author Christine Feehan."}], "Dark Fire (Feehan novel)": [null], "Passion (novel)": [{"Novels about legendary creatures and are also Paranormal romance, but not Contemporary fantasy.": "'''''Passion''''' is the third novel in the ''Fallen'' series written by Lauren Kate. It is a young adult, fantasy, paranormal romance published in 2011 under Delacorte Press. It continues the story of Lucinda Price who, at the end of Torment, decides to find out more about her past lives by stepping through an Announcer, ignoring Daniel's plea to stop. Daniel, a fallen angel, decides to follow her, promising to find and rescue her. Before Luce and Daniel met at Sword & Cross, before they fought the Immortals, they had already lived many lives. And so Luce, desperate to unlock the curse that condemns their love, must revisit her past incarnations in order to understand her fate. Each century, each life, holds a different clue. But Daniel is chasing her throughout the centuries before she has a chance to rewrite history."}], "Dark Celebration": [{"what are some Novels about legendary creatures and are also Paranormal romance": "is a paranormal/suspense novel"}], "Undead and Unwed": [{"Paranormal romance": "'''''Undead and Unwed''''' is a paranormal romance novel by MaryJanice Davidson.", "Novels about legendary creatures": "It is the first adventure of Elizabeth Anne \"Betsy\" Taylor in the Undead series after her transformation into a vampire.", "not Contemporary fantasy.": "'''''Undead and Unwed''''' is a paranormal romance novel by MaryJanice Davidson."}], "Dark Destiny": [{"what are some Novels about legendary creatures and are also Paranormal romance": "is a paranormal/suspense novel", "romance": "With the help of Nicolae's love she can finally be free from the vampires taint and be whole once more."}], "The Silver Kiss": [{"what are some Novels about legendary creatures and are also Paranormal romance, but not Contemporary fantasy.": "'''''The Silver Kiss''''' is a young adult, romance and horror novel written by Annette Curtis Klause; it is printed in hardcover and paperback versions. The novel was Klause's first; it was published on September 1, 1990, and was re-issued in 2009 with two bonus short stories by Klause. ''The Silver Kiss'' was inspired by Klause's poems and her teenage fantasy about romancing with a vampire."}], "Undead and Unappreciated": [{"what are some Novels about legendary creatures and are also Paranormal romance, but not Contemporary fantasy.": "The breezily written paranormal romance novel is told from the point of view of Betsy Taylor, reluctant Vampire Queen and not-so-reluctant shoe-hound, and introduces a major character to the series: Laura, Betsy's half-sister and the daughter of the Devil herself."}], "Dragon's Oath": [{"what are some Novels about legendary creatures and are also Paranormal romance, but not Contemporary fantasy.": "'''''Dragon's Oath''''' is the first of the ''House of Night'' novellas spin-off series, written by American authors P.C. Cast and Kristin Cast. It was published on July 12, 2011 by St. Martin's Press an extension of Macmillan Publishers.\n\n\nThe first book of the series takes place in Dragon's past, describing his meeting with Anastasia."}], "Dark Desire": [{"hat are some Novels about legendary creatures and are also Paranormal romance, but not Contemporary fantasy.": "'''''Dark Desire''''' is the second novel in Christine Feehan's Dark Series. It takes place roughly 25 years after the events in ''Dark Prince'' (1999).\nThe book begins with Jacques Dubrinsky, younger brother to Mikhail, being tortured and buried alive by members of the same fanatical group that attacked Raven and killed his sister some 25 years earlier in ''Dark Prince''. As time goes on he loses much of his memory (perhaps even his sanity), the only thing he has a clear memory of is of the faces of his human tormentors, and the fact that he was betrayed by someone close to him."}], "Dark Curse": [{"Novels about legendary creatures and are also Paranormal romance": "''''Dark Curse''''' is a 2008 dark fantasy vampire novel written by American author Christine Feehan, and a part of the ''Dark Series'' saga."}], "Dark Magic (novel)": [{"what are some Novels about legendary creatures and are also Paranormal romance, but not Contemporary fantasy.": "knows he's very close to turning into a monster. He has always known that a child of Mikhail will be his lifemate, so when Raven, Mikhail's lifemate, is attacked, he gives her his blood to build a bond between them."}], "Dark Guardian (novel)": [{"Novels": "'''''Dark Guardian''''' is a paranormal/suspense novel written by American author Christine Feehan.", "about legendary creatures and are also Paranormal romance": "Winner for 2002 Golden Rose Readers Choice Award from Love Romances for Best Vampire Romance"}], "Dark Secret (novel)": [{"Novels about legendary creatures and are also Paranormal romance": "'''''Dark Secret''''' is a novel written by American author Christine Feehan.\n\n\nRafael De La Cruz has spent centuries hunting vampires with his brothers, and with each passing year his capacity to feel emotions has grown weaker and weaker until finally there's barely been a memory left-until only sheer willpower keeps him from turning into the very abomination he hunts. But it'll take more than will to keep him away from the woman who is meant to be his and his alone..."}], "Dark Challenge": [{"Novels about legendary creatures and are also Paranormal romance,": "''''Dark Challenge''''' is the fifth book in the paranormal romance series Dark Series by American author Christine Feehan. It is the first book in a trilogy written within the Dark Series, and it starts several months after the events in ''Dark Magic''.\nJulian Savage, the twin brother of Aidan (''Dark Gold''), is sent to warn a young singer, Desari, that she and her band have come under suspicion by a fanatical vampire hunting society."}]}}} +{"query": "what are Fauna of Tibet that also are both Holarctic and Fauna of South India", "docs": ["Jerdon's baza", "Malayan night heron", "Mountain hawk-eagle", "Rufous-bellied eagle", "Spot-bellied eagle-owl", "Little spiderhunter", "Oriental dwarf kingfisher"], "original_query": "Fauna of Tibet that are also both Holarctic fauna and Fauna of South India", "scores": null, "metadata": {"template": "_ that are also both _ and _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "animals"}} +{"query": "Science books about cultural geography but not about cities", "docs": ["Geographia Neoteriki", "The Real Eve", "International Encyclopedia of Human Geography", "Hellenic Nomarchy", "America by Heart", "The Wealth and Poverty of Nations", "The Journey of Man", "Where Once We Walked", "The Old Neighborhood (book)", "Jewish Roots in Poland", "Freedom Rising", "Rural Rides", "Synopsis Universae Philologiae", "Searching for Whitopia", "The China\u2013Pakistan Axis", "The Nine Nations of North America", "Clash of Civilizations", "When Work Disappears"], "original_query": "Science books that are also Books about cultural geography but not Books about cities", "scores": null, "metadata": {"template": "_ that are also _ but not _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Geographia Neoteriki": ["Definitely relevant"], "The Real Eve": ["Definitely relevant"], "International Encyclopedia of Human Geography": ["Likely relevant"], "Hellenic Nomarchy": ["Likely relevant"], "America by Heart": ["Likely relevant"], "The Wealth and Poverty of Nations": ["Definitely relevant"], "The Journey of Man": ["Likely relevant"], "Where Once We Walked": ["Likely relevant"], "The Old Neighborhood (book)": ["Likely relevant"], "Jewish Roots in Poland": ["Likely relevant"], "Freedom Rising": ["Likely relevant"], "Rural Rides": ["Likely relevant"], "Synopsis Universae Philologiae": ["Definitely relevant"], "Searching for Whitopia": ["Definitely relevant"], "The China\u2013Pakistan Axis": ["Likely relevant"], "The Nine Nations of North America": ["Likely relevant"], "Clash of Civilizations": ["Likely relevant"], "When Work Disappears": ["Definitely relevant"]}, "evidence_ratings": {"Geographia Neoteriki": ["Partial"], "The Real Eve": ["Complete"], "International Encyclopedia of Human Geography": ["No evidence"], "Hellenic Nomarchy": ["Partial"], "America by Heart": ["No evidence"], "The Wealth and Poverty of Nations": ["Complete"], "The Journey of Man": ["Partial"], "Where Once We Walked": ["Partial"], "The Old Neighborhood (book)": ["Partial"], "Jewish Roots in Poland": ["Partial"], "Freedom Rising": ["Partial"], "Rural Rides": ["Partial"], "Synopsis Universae Philologiae": ["Partial"], "Searching for Whitopia": ["Complete"], "The China\u2013Pakistan Axis": ["Partial"], "The Nine Nations of North America": ["Partial"], "Clash of Civilizations": ["Partial"], "When Work Disappears": ["Complete"]}, "attributions": {"Geographia Neoteriki": [{"Science books about cultural geography": "'''''Geographia Neoteriki''''' ( ''Modern Geography'') is a geography book written in Greek by Daniel Philippidis and Grigorios Konstantas and printed in Vienna in 1791. It focused on both the physical and human geography features of the European continent and especially on Southeastern Europe, and is considered one of the most remarkable works of the modern Greek Enlightenment."}], "The Real Eve": [{"Science books": "'''''The Real Eve: Modern Man's Journey Out of Africa''''' is a popular science book about the evolution of modern humans written by British geneticist Stephen Oppenheimer.", "but not about cities": "The book is largely based on the \"out of Africa\" theory of human origins.", "about cultural geography": "Oppenheimer uses information from various disciplines including genetics, archaeology, anthropology and linguistics to synthesize theories on the origin of modern humans and their subsequent dispersal around the world."}], "International Encyclopedia of Human Geography": [null], "Hellenic Nomarchy": [{"about cultural geography": "It advocated the ideals of freedom, social justice and equality as the main principles of a well-governed society, making it the most important theoretical monument of Greek republicanism."}], "America by Heart": [null], "The Wealth and Poverty of Nations": [{"Science books about cultural geography but not about cities": "The book compared the long-term economic histories of different regions, specifically Europe, United States, Japan, China, the Arab world, and Latin America. In addition to analyzing economic and cliometric figures, he credited intangible assets, such as culture and enterprise, to explain economic success or failure."}], "The Journey of Man": [{"Science books about cultural geography but not about cities": "'''''The Journey of Man: A Genetic Odyssey''''' is a 2002 book by Spencer Wells, an American geneticist and anthropologist, in which he uses techniques and theories of genetics and evolutionary biology to trace the geographical dispersal of early human migrations out of Africa. The book was made into a TV documentary in 2003."}], "Where Once We Walked": [{"Science books about cultural geography but not about cities": "'''''Where Once We Walked''''' (full title: ''Where Once We Walked: A Guide to the Jewish Communities Destroyed in The Holocaust''), compiled by noted genealogist Gary Mokotoff and Sallyann Amdur Sack with Alexander Sharon, is a gazetteer of 37,000 town names in Central and Eastern Europe focusing on those with Jewish populations in the 19th and first half of the 20th centuries and most of whose Jewish communities were almost or completely destroyed during The Holocaust."}], "The Old Neighborhood (book)": [{"Science books about cultural geography but not about cities": "'''''The Old Neighborhood: What We Lost in the Great Suburban Migration: 1966-1999''''' is a 1999 non-fiction book by Ray Suarez. It describes the process of urban flight, as it has occurred in the United States from the 1960s to the 1990s."}], "Jewish Roots in Poland": [{"Science books about cultural geography but not about cities": "'''''Jewish Roots in Poland''''' (full title: ''Jewish Roots in Poland: Pages from the Past and Archival Inventories'') is a book created by genealogist Miriam Weiner and co-published by The Miriam Weiner Routes to Roots Foundation and YIVO Institute for Jewish Research. A searchable database of updated archival holdings listed in the book is available in the Archive Database on the Routes to Roots Foundation website."}], "Freedom Rising": [{"Science books about cultural geography but not about cities": "'''''Freedom Rising: Human Empowerment and the Quest for Emancipation''''' is a 2013 book by the German political scientist Christian Welzel"}], "Rural Rides": [{"Science books about cultural geography": "'''''Rural Rides''''' is the book for which the English journalist, agriculturist and political reformer William Cobbett is best known."}], "Synopsis Universae Philologiae": [{"Science books about cultural geography but not about cities": "The four ''mappae geographico-polyglottae'' give the beginning of the ''pater noster'' in various languages and scripts."}], "Searching for Whitopia": [{"Science books about cultural geography but not about cities": "'''''Searching for Whitopia: An Improbable Journey to the Heart of White America''''' is a 2009 non-fiction book by Rich Benjamin.\n\n\nIn May 2010, Benjamin briefly summarized his experiences in a TED talk.\nAfrican American journalist Rich Benjamin documents his journeys to find out why more and more white Americans move to small towns and areas that are, for the most part, white, and to explain why Whitopias are growing and what it means for the United States."}], "The China\u2013Pakistan Axis": [{"cultural geography": "'''''The China\u2013Pakistan Axis: Asia's New Geopolitics''''' is a book by the British author Andrew Small exploring the cultural and political ties as well as depicting the relationship between Pakistan and China, released in early 2015.", "Science": "He previously worked as the director of the Foreign Policy Centre's Beijing office; as a visiting fellow at the Chinese Academy of Social Sciences, and was an ESU scholar in the office of Senator Edward M. Kennedy."}], "The Nine Nations of North America": [{"cultural geography": "'''''The Nine Nations of North America''''' is a 1981 book by Joel Garreau, in which the author suggests that North America can be divided into nine nations, which have distinctive economic and cultural features."}], "Clash of Civilizations": [{"Science books about cultural geography": "The '''Clash of Civilizations''' is a thesis that people's cultural and religious identities will be the primary source of conflict in the post\u2013Cold War world."}], "When Work Disappears": [{"Science books about cultural geography": "'''''When Work Disappears: The World of the New Urban Poor''''' (1996) is a book by William Julius Wilson, Professor of Social Policy at Harvard. Wilson's argument is that the disappearance of work and the consequences of that disappearance for both social and cultural life are the central problems in the inner-city ghetto. He sought to discuss social disorganization without stigmatizing the poor. Wilson writes that chronic joblessness has deprived those in the inner city of skills necessary to obtain and keep jobs. Wilson's book uses evidence from large-scale scientific surveys in the ghetto and information culled from ethnographic interviews of ghetto residents in order to create a complete picture of the problems that face the residents."}]}}} +{"query": "British War films based on actual events shot in the Republic of Ireland", "docs": ["Saving Private Ryan", "My Boy Jack (film)"], "original_query": "British films that are also both War films based on actual events and Films shot in the Republic of Ireland", "scores": null, "metadata": {"template": "_ that are also both _ and _", "domain": "films", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Saving Private Ryan": ["Likely relevant"], "My Boy Jack (film)": ["Likely relevant"]}, "evidence_ratings": {"Saving Private Ryan": ["Partial"], "My Boy Jack (film)": ["Partial"]}, "attributions": {"Saving Private Ryan": [{"British War films based on actual events shot in the Republic of Ireland": "Filming began June 27, 1997, and lasted for two months. Spielberg wanted an almost exact replica of the Omaha Beach landscape for the movie, including sand and a bluff similar to the one where German forces were stationed and a near match was found in Ballinesker Beach, Curracloe Strand, Ballinesker, just east of Curracloe, County Wexford, Ireland."}], "My Boy Jack (film)": [{"British War films based on actual events shot in the Republic of Ireland": "'''''My Boy Jack''''' is a 2007 British biographical television film based on David Haig's 1997 play of the same name for ITV. It was filmed in August 2007, with Haig as Rudyard Kipling and Daniel Radcliffe as John Kipling. The American television premiere was on 20 April 2008 on PBS, with primetime rebroadcast on 27 March 2011. The film attracted about 5.7 million viewers on its original ITV broadcast in the UK on Remembrance Day, 11 November 2007."}]}}} +{"query": "Flora of Costa Rica and Brassicaceae genera", "docs": ["Pennellia", "Romanschulzia"], "original_query": "Brassicaceae genera that are also Flora of Costa Rica", "scores": null, "metadata": {"template": "_ that are also _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "plants"}} +{"query": "American books based on the Bible but not about Judaism.", "docs": ["Let My People Go: Bible Stories Told By A Freeman Of Color", "The Wonder Book of Bible Stories", "Hadassah: One Night with the King", "Live from Golgotha: The Gospel According to Gore Vidal", "The Tables of the Law", "Sarah (Card novel)", "Stone Tables", "The Prince (novel)", "The Judas Testament (novel)", "Animals of the Bible", "Rebekah (novel)", "The Wives of Israel", "Silverlock", "Redeeming Love", "Seola", "Slaves of the Shinar", "The Lonely Man of Faith", "Rachel and Leah"], "original_query": "American books that are also Books based on the Bible but not Books about Jews and Judaism", "scores": null, "metadata": {"template": "_ that are also _ but not _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Let My People Go: Bible Stories Told By A Freeman Of Color": ["Definitely relevant"], "The Wonder Book of Bible Stories": ["Definitely relevant"], "Hadassah: One Night with the King": ["Likely relevant"], "Live from Golgotha: The Gospel According to Gore Vidal": ["Likely relevant"], "The Tables of the Law": ["Definitely relevant"], "Sarah (Card novel)": ["Definitely relevant"], "Stone Tables": ["Definitely relevant"], "The Prince (novel)": ["Definitely relevant"], "The Judas Testament (novel)": ["Definitely relevant"], "Animals of the Bible": ["Definitely relevant"], "Rebekah (novel)": ["Likely relevant"], "The Wives of Israel": ["Definitely relevant"], "Silverlock": ["Likely relevant"], "Redeeming Love": ["Definitely relevant"], "Seola": ["Definitely relevant"], "Slaves of the Shinar": ["Likely relevant"], "The Lonely Man of Faith": ["Definitely relevant"], "Rachel and Leah": ["Definitely relevant"]}, "evidence_ratings": {"Let My People Go: Bible Stories Told By A Freeman Of Color": ["Complete"], "The Wonder Book of Bible Stories": ["Complete"], "Hadassah: One Night with the King": ["Partial"], "Live from Golgotha: The Gospel According to Gore Vidal": ["Partial"], "The Tables of the Law": ["Complete"], "Sarah (Card novel)": ["Complete"], "Stone Tables": ["Complete"], "The Prince (novel)": ["Complete"], "The Judas Testament (novel)": ["Complete"], "Animals of the Bible": ["Complete"], "Rebekah (novel)": ["Partial"], "The Wives of Israel": ["Complete"], "Silverlock": ["No evidence"], "Redeeming Love": ["Complete"], "Seola": ["Partial"], "Slaves of the Shinar": ["Partial"], "The Lonely Man of Faith": ["Complete"], "Rachel and Leah": ["Complete"]}, "attributions": {"Let My People Go: Bible Stories Told By A Freeman Of Color": [{"American books based on the Bible but not about Judaism.": "'''''Let My People Go: Bible Stories Told By A Freeman Of Color''''' is a 1998 book by Patricia McKissack. Set in 19th century South Carolina, it is about a freed slave, Price Jeffries, who uses Bible stories from the Old Testament to answer questions that his daughter, Charlotte, poses about the things she sees around her."}], "The Wonder Book of Bible Stories": [{"American books based on the Bible": "'''''The Wonder Book of Bible Stories''''' is a 1904 collection by Logan Marshall published in the United States. The book includes biblical stories from both the Old and New Testaments retold for children and illustrated with rich woodcuts and color plates."}], "Hadassah: One Night with the King": [{"books based on the Bible": "'''''Hadassah: One Night with the King''''' is a 2004 novel by Tommy Tenney and Mark Andrew Olsen based upon a retelling of the Biblical Book of Esther."}], "Live from Golgotha: The Gospel According to Gore Vidal": [{"American books based on the Bible but not about Judaism.": "'''''Live from Golgotha: The Gospel according to Gore Vidal''''' is a novel by Gore Vidal, an irreverent spoof of the New Testament. Told from the perspective of Saint Timothy as he travels with Saint Paul, the 1992 novel's narrative shifts in time as Timothy and Paul combat a mysterious hacker from the future who is deleting all traces of Christianity."}], "The Tables of the Law": [{"American books based on the Bible but not about Judaism.": "'''''The Tables of the Law''''' () is a 1944 novella by German writer Thomas Mann. It is a dramatic retelling of the Biblical story of Moses contained in the Book of Exodus, although some of the laws which Moses proscribes for his followers are taken from Leviticus"}], "Sarah (Card novel)": [{"American books based on the Bible but not about Judaism.": "'''''Sarah: Women of Genesis''''' (2000) is the first novel in the ''Women of Genesis'' series by Orson Scott Card.\n''Sarah'' follows the story of Abraham through the eyes and perspective of Sarah. The Biblical account of the life of Sarah is contained in Genesis 12 - 22 (about 16 pages) most of which is centered on Abraham."}], "Stone Tables": [{"American books based on the Bible": "'''''Stone Tables''''' (1997) is a historical novel by Orson Scott Card, based on the life of Moses. As with much of Card's other literature, a Christian/Mormon influence is present in this book."}], "The Prince (novel)": [{"American books based on the Bible but not about Judaism.": "'''''The Prince''''' is an American novel published in 2005, written by Francine Rivers. It is the third novel in the Sons of Encouragement series, and tells the tale of the biblical character of Jonathan, the son of Saul the King in the Old Testament."}], "The Judas Testament (novel)": [{"American books based on the Bible but not about Judaism.": "'''''The Judas Testament''''' is a 1994 novel by Daniel Easterman. The plot revolved around the discovery of an ancient scroll hidden deep in the vaults of the Russian State Library in Moscow, and its translation by Dr. Jack Gould, a scholar of the Aramaic language, who becomes the unwitting pawn in a murderous struggle by various crypto-political forces to possess the scroll."}], "Animals of the Bible": [{"American books based on the Bible but not about Judaism.": "''Animals of the Bible'' takes 31 Biblical stories of creatures who helped Biblical heroes complete their tasks and illustrates them."}], "Rebekah (novel)": [{"based on the Bible": "The story-line does not deviate from the story told in Genesis, but Card does add details and characters of his own invention."}], "The Wives of Israel": [{"American books based on the Bible but not about Judaism.": "'''''The Wives of Israel''''' will be the fourth novel in the ''Women of Genesis'' series by Orson Scott Card. It has yet to be released.\nThe book will be a continuation of the previous novel in the ''Women of Genesis'' series ''Rachel and Leah''. The book left off after Rachel had married Jacob, following the marriage of her sister Leah. Card states in the afterword of ''Rachel and Leah'' that he had not intended to have the story be continued in two more books, but that it would have been too much to include the marriage/concubinage of the sister's handmaidens and so decided to put them into consecutive books to cover that subject as well as the children and their raising and adventures (chiefly Joseph and his brothers, and Joseph's further adventures into slavery in Egypt, possibly from the point of his wife). Though Card says that he plans to leave the matter of Judah's daughter-in-law, Tamar, alone.\nAt this time, the expected release date is unknown."}], "Silverlock": [{"American books based on the Bible": "'''''Silverlock''''' is a novel by John Myers Myers published in 1949. The novel's settings and characters, aside from the protagonist, are all drawn from history, mythology, and other works of literature."}], "Redeeming Love": [{"American books based on the Bible but not about Judaism.": "'''''Redeeming Love''''' is a historical romance novel by Francine Rivers set in the 1850s Gold Rush in California. The story is inspired by the Book of Hosea from the Bible, and its central theme is the redeeming love of God towards sinners."}], "Seola": [{"American books based on the Bible but not about Judaism.": "'''''Seola''''' is an antediluvian novel published in 1878, written by Ann Eliza Smith. The publishers of the novel are Boston: Lee and Shepard, New York: Charles T. Dillingham.\n\n\nThe majority of the novel purports to be a translation of an ancient scroll diary written by a woman named Seola, who is identified as the wife of Japheth. The Book of Genesis indicates that Noah had three sons named Ham, Shem and Japheth. In the appendix section of the novel, Ann Smith describes how she was inspired to write the fantasy. She writes:"}], "Slaves of the Shinar": [{"books based on the Bible but not about Judaism.": "The book explores the early lives and origins of characters and peoples to be found in Middle Eastern myths and religious texts, especially the Biblical Book of Genesis and the ''Epic of Gilgamesh''."}], "The Lonely Man of Faith": [{"American books based on the Bible but not about Judaism.": "'''''The Lonely Man of Faith''''' is a philosophical essay written by Rabbi Joseph B. Soloveitchik, first published in the summer 1965 issue of Tradition, as a book by Doubleday in 1992 and published in a newly revised edition in 2011 by Koren Publishers Jerusalem."}], "Rachel and Leah": [{"based on the Bible": "follows the story of Jacob through the eyes of Rachel and Leah.", "but not about Judaism.": "the storyline does not deviate from the story told in Genesis."}]}}} +{"query": "Series of Children's books that are Novellas", "docs": ["Sir Harold and the Gnome King", "The Short Second Life of Bree Tanner", "Bloodstained Oz"], "original_query": "Series of children's books that are also both Novellas and Children's books", "scores": null, "metadata": {"template": "_ that are also both _ and _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Sir Harold and the Gnome King": ["Definitely relevant"], "The Short Second Life of Bree Tanner": ["Definitely relevant"], "Bloodstained Oz": ["Definitely relevant"]}, "evidence_ratings": {"Sir Harold and the Gnome King": ["Partial"], "The Short Second Life of Bree Tanner": ["Partial"], "Bloodstained Oz": ["Partial"]}, "attributions": {"Sir Harold and the Gnome King": [{"Series of Children's books that are Novellas": "'''''Sir Harold and the Gnome King''''' is a fantasy novella American writer L. Sprague de Camp, part of the Harold Shea series he originated in collaboration with Fletcher Pratt and later continued with Christopher Stasheff."}], "The Short Second Life of Bree Tanner": [{"Series of Children's books that are Novellas": "''''The Short Second Life of Bree Tanner''''' is a novella in the ''Twilight'' series by author Stephenie Meyer."}], "Bloodstained Oz": [{"Series of Children's books that are Novellas": "''''Bloodstained Oz''''' is a ''Wizard of Oz'' related novella by Christopher Golden and James A. Moore, and it was illustrated by Glenn Chadbourne."}]}}} +{"query": "Angiosperm genera of France", "docs": ["Neogaya", "Murbeckiella", "Endressia", "Odontarrhena", "Schlagintweitia", "Xatardia"], "original_query": "Angiosperm genera that are also Flora of France", "scores": null, "metadata": {"template": "_ that are also _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "plants"}} +{"query": "Aquatic reptiles that aren't from the prehistoric time", "docs": ["Acrochordoidea", "Mangrove monitor", "Tchoiria", "Liaoxisaurus", "Saltwater crocodile", "Askeptosaurus", "Augustasaurus", "Wumengosaurus", "Suwannee snapping turtle", "Gwyneddosaurus", "Apalachicola snapping turtle", "Marine iguana", "Alligator snapping turtle", "Pistosaurus", "Nanchangosaurus"], "original_query": "Aquatic reptiles that are not Prehistoric marine crocodylomorphs", "scores": null, "metadata": {"template": "_ that are not _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "animals"}} +{"query": "1961 historical novels", "docs": ["The Golden Goblet", "The Old Men at the Zoo", "Return from the Stars", "Friedrich (novel)", "Dawn Wind"], "original_query": "Historical novels subgenres that are also 1961 books", "scores": null, "metadata": {"template": "_ that are also _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"The Golden Goblet": ["Definitely relevant"], "The Old Men at the Zoo": ["Definitely relevant"], "Return from the Stars": ["Definitely relevant"], "Friedrich (novel)": ["Likely relevant"], "Dawn Wind": ["Definitely relevant"]}, "evidence_ratings": {"The Golden Goblet": ["Complete"], "The Old Men at the Zoo": ["Complete"], "Return from the Stars": ["Partial"], "Friedrich (novel)": ["Partial"], "Dawn Wind": ["Complete"]}, "attributions": {"The Golden Goblet": [{"1961 historical novels": "'''''The Golden Goblet''''' is a children's historical novel by Eloise Jarvis McGraw. It was first published in 1961 and received a Newbery Honor award in 1962."}], "The Old Men at the Zoo": [{"1961 historical novels": "'''''The Old Men at the Zoo''''' is a novel written by Angus Wilson, first published in 1961 by Secker and Warburg and by Penguin books in 1964. It was adapted, with many changes\u2014nuclear bombing of London, not present in the novel, is added\u2014into a 1983 BBC Television serial by the scriptwriter Troy Kennedy Martin. The book deals with events before a nuclear attack on London during a (presumably) limited nuclear war, which results in the imposition of a later post-apocalyptic pan-European dystopian dictatorship, until rescue arrives for the prisoners at the zoo, transformed into a concentration camp."}], "Return from the Stars": [{"1961 historical novels": "'''''Return from the Stars''''' () is a science fiction novel by Polish author Stanis\u0142aw Lem. Written in 1961, it is the story of a cosmonaut returning to his homeworld, Earth, after more than a century in Earth time, but just 10 years for him, finding it a completely different place, with many developments he dislikes"}], "Friedrich (novel)": [{"1961": "It is by the author Hans Peter Richter. ''Friedrich'' was first printed in 1961."}], "Dawn Wind": [{"1961 historical novel": "'''''Dawn Wind''''' is a historical novel for children and young adults written by Rosemary Sutcliff and published in 1961 by Oxford University Press, with illustrations by Charles Keeping."}]}}} +{"query": "Supernatural thriller films shot in Andhra Pradesh", "docs": ["Karthikeya (film)", "Maaya (2014 film)"], "original_query": "Films shot in Andhra Pradesh that are also Supernatural thriller films", "scores": null, "metadata": {"template": "_ that are also _", "domain": "films", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Karthikeya (film)": ["Definitely relevant"], "Maaya (2014 film)": ["Definitely relevant"]}, "evidence_ratings": {"Karthikeya (film)": ["Complete"], "Maaya (2014 film)": ["Complete"]}, "attributions": {"Karthikeya (film)": [{"Supernatural thriller films shot in Andhra Pradesh": "'''''Karthikeya''''' is a 2014 Indian Telugu-language supernatural mystery thriller film written and directed by Chandoo Mondeti. It was produced by Venkat Srinivas on Magnus Cine Prime Pvt. Ltd. and featured Nikhil Siddharth, Swati Reddy, Tanikella Bharani, Rao Ramesh and Jayaprakash in important roles. Shekar Chandra composed the film's music while Karthik Ghattamaneni and Karthika Srinivas were its cinematographer and editor respectively. The film revolves around a closed Kartikeya temple on the border of Andhra Pradesh and Tamil Nadu and showcases the life of a curious medico Karthik and his nexus with the temple."}], "Maaya (2014 film)": [{"Supernatural thriller films": "'''''Maaya''''' is a 2014 Telugu psychological supernatural thriller film", "shot in Andhra Pradesh": "The film was shot in Hyderabad and in and around Andhra Pradesh and Telangana."}]}}} +{"query": "Futuristic films set in the Southwestern United States about Marines", "docs": ["Battle: Los Angeles", "Independence Day: Resurgence"], "original_query": "Films about marines that are also both Films set in the future and Films set in the Southwestern United States", "scores": null, "metadata": {"template": "_ that are also both _ and _", "domain": "films", "fluency": ["Mostly Fluent: It has a few errors or it does not sound natural, but I can understand it."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Maybe - The query expresses a niche interest, but a user could potentially issue such a query."], "relevance_ratings": {"Battle: Los Angeles": ["Likely relevant"], "Independence Day: Resurgence": ["Likely relevant"]}, "evidence_ratings": {"Battle: Los Angeles": ["Partial"], "Independence Day: Resurgence": ["Partial"]}, "attributions": {"Battle: Los Angeles": [{"set in the Southwestern United States about Marines": "The film is set in modern-day Los Angeles and follows a Marine Staff Sergeant played by Aaron Eckhart who leads a platoon of U.S. Marines, joined by other stranded military personnel, during a global alien invasion."}], "Independence Day: Resurgence": [{"Futuristic films": "The film takes place twenty years after the events of the first film, during which the United Nations has collaborated to form the Earth Space Defense, an international military defense and research organization.", "Southwestern United States": ". Principal photography began in April 2015 at locations primarily in New Mexico but also in the Bonneville Salt Flats of Utah, which were featured in the original. This was one of Robert Loggia's final projects before he died and the film is dedicated in memory of him."}]}}} +{"query": "Non-American animated superhero films from the 2010s", "docs": ["Promare", "Henchmen (film)", "Batman Unlimited: Monster Mayhem", "Suicide Squad: Hell to Pay", "DC Showcase: Green Arrow", "DC Showcase: Catwoman", "StarDog and TurboCat", "3 Bahadur: Rise of the Warriors", "The Lego Ninjago Movie", "Chakra: The Invincible"], "original_query": "2010s animated superhero films that are not American animated superhero films", "scores": null, "metadata": {"template": "_ that are not _", "domain": "films", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Promare": ["Likely relevant"], "Henchmen (film)": ["Definitely relevant"], "Batman Unlimited: Monster Mayhem": ["Definitely relevant"], "Suicide Squad: Hell to Pay": ["Definitely relevant"], "DC Showcase: Green Arrow": ["Likely relevant"], "DC Showcase: Catwoman": ["Likely relevant"], "StarDog and TurboCat": ["Definitely relevant"], "3 Bahadur: Rise of the Warriors": ["Definitely relevant"], "The Lego Ninjago Movie": ["Likely relevant"], "Chakra: The Invincible": ["Definitely relevant"]}, "evidence_ratings": {"Promare": ["Complete"], "Henchmen (film)": ["Partial"], "Batman Unlimited: Monster Mayhem": ["Complete"], "Suicide Squad: Hell to Pay": ["No evidence"], "DC Showcase: Green Arrow": ["Partial"], "DC Showcase: Catwoman": ["Partial"], "StarDog and TurboCat": ["Complete"], "3 Bahadur: Rise of the Warriors": ["Complete"], "The Lego Ninjago Movie": ["Partial"], "Chakra: The Invincible": ["Complete"]}, "attributions": {"Promare": [{"Non-American animated superhero films from the 2010s": "a 2019 Japanese animated film", "superhero": "Kray informs Galo that the Earth will soon be destroyed by an uncontrollable surge of magma from the earth's core. He reveals that the abilities of the Burnish can be used to create a warp drive, which he intends to use to flee Earth with a select portion of humanity"}], "Henchmen (film)": [{"Non-American animated superhero films from the 2010s": "'''''Henchmen''''' is a 2018 Canadian computer-animated action comedy film directed by Adam Wood and co-written by Wood, Jay D. Waxman, David Ray, and Bobby Henwood, based on Wood's own 2014 short film ''Henchmen: Ill"}], "Batman Unlimited: Monster Mayhem": [{"Non-American animated superhero films from the 2010s": "'''''Batman Unlimited: Monster Mayhem''''' is a direct-to-video animated superhero film and the second entry in the ''Batman Unlimited'' series. It was released on August 18, 2015 on Blu-ray, DVD and Digital HD. It is a sequel to ''Batman Unlimited: Animal Instincts''."}], "Suicide Squad: Hell to Pay": [null], "DC Showcase: Green Arrow": [{"animated superhero films from the 2010s": "'''''DC Showcase: Green Arrow''''' is a 2010 short animated film, directed by Joaquim Dos Santos and written by Greg Weisman, featuring Neal McDonough as Green Arrow, who must protect a young princess at an airport."}], "DC Showcase: Catwoman": [{"Non-American animated superhero films from the 2010s": "'''''DC Showcase: Catwoman''''', also titled as simply ''Catwoman'', is a 2011 short animated superhero film"}], "StarDog and TurboCat": [{"Non-American animated superhero films from the 2010s": "'''''StarDog and TurboCat''''' is a 2019 British 3D computer-animated superhero film"}], "3 Bahadur: Rise of the Warriors": [{"Non-American animated superhero films from the 2010s": "''''3 Bahadur: Rise of the Warriors''''' () is a Pakistani computer-animated action film d", "2010s": "''3 Bahadur: Rise of the Warriors'' released on 14 December 2018 by ARY Films."}], "The Lego Ninjago Movie": [{"from the 2010s": "'''''The Lego Ninjago Movie''''' is a 2017 computer-animated martial arts adventure-comedy film based on the toy line of the same name and the tie-in animated television series ''Ninjago: Masters of Spinjitzu''.", "animated": "'''''The Lego Ninjago Movie''''' is a 2017 computer-animated martial arts adventure-comedy film based on the toy line of the same name and the tie-in animated television series ''Ninjago: Masters of Spinjitzu''.", "superhero films": "The film focuses on a young teenage ninja Lloyd Garmadon, as he attempts to accept the truth about his villainous father while a new threat emerges to endanger his homeland and learn to became a true ninja warrior."}], "Chakra: The Invincible": [{"Non-American animated superhero films": "an Indian animated superhero film", "from the 2010s": "premiered on Cartoon Network on 30 November 2013."}]}}} +{"query": "Arecaceae that are trees of Indo-China", "docs": ["Coconut", "Eleiodoxa"], "original_query": "Arecaceae that are also Trees of Indo-China", "scores": null, "metadata": {"template": "_ that are also _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "plants"}} +{"query": "2007 non Indian Indigenous drama films", "docs": ["Mongol (film)", "Bury My Heart at Wounded Knee (film)", "Genghis Khan: To the Ends of the Earth and Sea", "The Bucket List", "Pure Coolness", "Pathfinder (2007 film)", "Alexandra (film)", "Ca\u00f3tica Ana"], "original_query": "Indigenous films that are also 2007 drama films but not Indian drama films", "scores": null, "metadata": {"template": "_ that are also _ but not _", "domain": "films", "fluency": ["Mostly Fluent: It has a few errors or it does not sound natural, but I can understand it."], "meaning": ["Too Ambiguous: It's too ambiguous to make a reasonable judgement. Under some likely interpretations of the two queries they have the same meaning, but under other likely interpretations they have different meanings."], "naturalness": ["Maybe - The query expresses a niche interest, but a user could potentially issue such a query."], "relevance_ratings": {"Mongol (film)": ["Likely relevant"], "Bury My Heart at Wounded Knee (film)": ["Definitely relevant"], "Genghis Khan: To the Ends of the Earth and Sea": ["Definitely relevant"], "The Bucket List": ["Likely relevant"], "Pure Coolness": ["Likely relevant"], "Pathfinder (2007 film)": ["Likely relevant"], "Alexandra (film)": ["Likely relevant"], "Ca\u00f3tica Ana": ["Definitely relevant"]}, "evidence_ratings": {"Mongol (film)": ["Partial"], "Bury My Heart at Wounded Knee (film)": ["Complete"], "Genghis Khan: To the Ends of the Earth and Sea": ["Complete"], "The Bucket List": ["Partial"], "Pure Coolness": ["Partial"], "Pathfinder (2007 film)": ["Partial"], "Alexandra (film)": ["Partial"], "Ca\u00f3tica Ana": ["Complete"]}, "attributions": {"Mongol (film)": [{"2007 non Indian Indigenous drama films": "\"The sweeping ''Mongol'' mixes romance, family drama, and enough flesh-ripping battle scenes to make sense of Ghenghis Khan's legendary stature."}], "Bury My Heart at Wounded Knee (film)": [{"2007 non Indian Indigenous drama films": "'''''Bury My Heart at Wounded Knee''''' is a 2007 American Western historical drama television film adapted from the 1970 book of the same name by Dee Brown.", "Indigenous": "The book on which the film is based is a history of Native Americans in the American West in the 1860s and 1870s, focusing upon the transition from traditional ways of living to living on reservations and their treatment during that period."}], "Genghis Khan: To the Ends of the Earth and Sea": [{"2007 non Indian": "is a 2007 Japanese \u2013 Mongolian historical drama film depicting the life of Genghis Khan.", "drama films": "is a 2007 Japanese \u2013 Mongolian historical drama film depicting the life of Genghis Khan.", "Indigenous": "Temujin (Takashi Sorimachi) is born to the chief of a Mongolian tribe, and grows up as the one who carries the blood of \u201cblue wolf\u201d."}], "The Bucket List": [{"2007 non Indian Indigenous drama films": "'''''The Bucket List''''' is a 2007 American buddy adventure comedy-drama film directed and produced by Rob Reiner, written by Justin Zackham, and starring Jack Nicholson and Morgan Freeman. The main plot follows two terminally ill men on their road trip with a wish list of things to do before they \"kick the bucket\"."}], "Pure Coolness": [{"2007 non Indian Indigenous drama films": "'''''Pure Coolness''''' () is a 2007 film by Kyrgyz director Ernest Abdyjaparov.", "Indigenous": "The film is about the traditional custom of Ala kachuu \u2013 \"bride stealing\". The film addresses the question of whether the practice can be defended in this day and age, even if the couple end up living \"happily ever after\"."}], "Pathfinder (2007 film)": [{"2007 non Indian Indigenous drama films": "'''''Pathfinder''''' (also known by the alternate title '''''Pathfinder: The Legend of the Ghost Warrior''''') is a 2007 American epic action film", "Indigenous": "A Viking Age expedition arrives in North America, intending to subjugate or slaughter the native \"Skr\u00e6ling\" population."}], "Alexandra (film)": [{"2007 non Indian Indigenous drama films": "'''''Alexandra''''' (, translit.\u00a0Aleksandra) is a 2007 Russian film about the Second Chechen War"}], "Ca\u00f3tica Ana": [{"2007 non Indian Indigenous drama films": "'''''Ca\u00f3tica Ana''''' is a 2007 drama film by Spanish director Julio M\u00e9dem."}]}}} +{"query": "Novels set in elementary and primary schools excluding American children's novels", "docs": ["From Anna", "Totto-Chan: The Little Girl at the Window", "The Honours Board", "Gentlemen & Players", "Mine for Keeps", "The Turbulent Term of Tyke Tiler", "Heart (novel)", "Our Twisted Hero", "Botchan", "More Sideways Arithmetic from Wayside School", "Gowie Corby Plays Chicken", "The Harpole Report", "Bill's New Frock"], "original_query": "Novels set in elementary and primary schools that are not American children's novels", "scores": null, "metadata": {"template": "_ that are not _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"From Anna": ["Definitely relevant"], "Totto-Chan: The Little Girl at the Window": ["Definitely relevant"], "The Honours Board": ["Definitely relevant"], "Gentlemen & Players": ["Definitely relevant"], "Mine for Keeps": ["Likely relevant"], "The Turbulent Term of Tyke Tiler": ["Definitely relevant"], "Heart (novel)": ["Definitely relevant"], "Our Twisted Hero": ["Definitely relevant"], "Botchan": ["Definitely relevant"], "More Sideways Arithmetic from Wayside School": ["Likely relevant"], "Gowie Corby Plays Chicken": ["Definitely relevant"], "The Harpole Report": ["Definitely relevant"], "Bill's New Frock": ["Likely relevant"]}, "evidence_ratings": {"From Anna": ["Complete"], "Totto-Chan: The Little Girl at the Window": ["Complete"], "The Honours Board": ["Complete"], "Gentlemen & Players": ["Complete"], "Mine for Keeps": ["Partial"], "The Turbulent Term of Tyke Tiler": ["Complete"], "Heart (novel)": ["Complete"], "Our Twisted Hero": ["Partial"], "Botchan": ["Partial"], "More Sideways Arithmetic from Wayside School": ["Partial"], "Gowie Corby Plays Chicken": ["Complete"], "The Harpole Report": ["Complete"], "Bill's New Frock": ["Complete"]}, "attributions": {"From Anna": [{"excluding American children's novels": "'''''From Anna''''' is a children's novel written by Canadian children's author Jean Little, first published in 1972. It is the story of Anna Solden, a visually impaired child who moves from Germany to Canada with her family, on the eve of Hitler's rise to power in Germany.", "Novels set in elementary and primary schools": "The new headmaster at the Solden's school forbids the singing of a song titled \"My thoughts are free\" (Die Gedanken sind frei) during an assembly, instead making the school sing the national anthem."}], "Totto-Chan: The Little Girl at the Window": [{"set in elementary": "a Tokyo elementary school founded by educator Sosaku Kobayashi during World War II."}], "The Honours Board": [{"Novels set in elementary and primary schools excluding American children's novels": "''''The Honours Board''''' is a novel by Pamela Hansford Johnson first published in 1970. Set in the South of England at Downs Park, a small fictional preparatory school for boys, it follows the lives of the members of the staff over a couple of years."}], "Gentlemen & Players": [{"Novels set in elementary and primary schools": "Harris' experiences as a teacher at Leeds Grammar School. Set in the present day during Michaelmas term at St Oswald's, a grammar school for boys somewhere in the North of England"}], "Mine for Keeps": [{"Novels set in elementary and primary schools excluding American children's novels": "'''''Mine for Keeps''''' is a 1962 book by the Canadian children's author Jean Little. At the time she wrote ''Mine for Keeps'', Little was teaching in a school for the disabled and she had written the book after becoming tired of reading her students books in which disabled child characters either meet deaths or recover completely (like Clara in ''Heidi'', or Colin in ''The Secret Garden'')."}], "The Turbulent Term of Tyke Tiler": [{"Novels set in elementary and primary schools excluding American children's novels": "'''''The Turbulent Term of Tyke Tiler''''' (or '''''Tyke Tiler''''') is a children's school adventure novel by Gene Kemp, first published by Faber and Faber in 1977 with illustrations by Carolyn Dinan. It is set at Cricklepit Combined School, a fictional primary school based on St Sidwell's School in Exeter where Kemp worked as a teacher from 1963 to 1979."}], "Heart (novel)": [{"Novels set in elementary and primary schools excluding American children's novels": "Cover of ''Herz'', an 1894 German translation.\n'''''Heart''''' ( ) is a children's novel by the Italian author Edmondo De Amicis who was a novelist, journalist, short story writer, and poet. The novel is his best known work to this day, having been inspired by his own children Furio and Ugo who had been schoolboys at the time. It is set during the Italian unification, and includes several patriotic themes. It was issued by Treves on October 18, 1886, the first day of school in Italy, and rose to immediate success."}], "Our Twisted Hero": [{"Novels set in elementary and primary schools excluding American children's novels": "* Pyongtae's Wife\nThis story is told by a man named Han Byeong-tae (or Py\u014fngt'ae), recalling his memories when he was in 5th grade and part of 6th grade.\n\n\nDue to Byeong-tae's father failing in business, they move to a low town and go to Y Elementary School."}], "Botchan": [{"Novels set in elementary and primary schools": "In the opening chapter he hurts himself jumping from the second floor of his elementary school, fights the boy next door, and tramples a neighbor's carrot patch by wrestling (sumo style) on the straw that covers the seedlings."}], "More Sideways Arithmetic from Wayside School": [{"Novels set in elementary and primary schools": "'''''More Sideways Arithmetic From Wayside School''''' is a children's novel by Louis Sachar in the ''Wayside School'' series."}], "Gowie Corby Plays Chicken": [{"Novels set in elementary and primary schools": "'''''Gowie Corby Plays Chicken''''' () is a children's novel by Gene Kemp, set at the fictional Cricklepit Combined primary school in southern England."}], "The Harpole Report": [{"Novels set in elementary and primary schools": "The novel tells the story mostly in the form of a school log book kept by George Harpole, temporary Head Teacher of the Church of England primary school of \"Tampling St. Nicholas\"."}], "Bill's New Frock": [{"Novels set in elementary and primary schools": "Now forced to go to school in a pink dress, Bill discovers one of the worst days in his life is about to unfold. Baffled by the way things are just different for girls, Bill falls headlong into trouble. The book was adapted into a television special, which first aired on 6 June 1998."}]}}} +{"query": "Birds found in South China and Vietnam, but not in Yunnan", "docs": ["Pied falconet", "Kloss's leaf warbler", "Yellow-billed nuthatch", "Fork-tailed sunbird", "Greyish-crowned leafbird", "Russet bush warbler", "Hodgson's hawk-cuckoo", "Blue-rumped pitta", "White-bellied green pigeon", "Ratchet-tailed treepie", "Cook's swift", "Spectacled barwing", "White-eared night heron", "Rufous-capped babbler", "Crested kingfisher", "Grey-headed parrotbill", "White-shouldered starling", "Indochinese green magpie", "Streak-breasted scimitar babbler"], "original_query": "Birds of South China that are also Birds of Vietnam but not Birds of Yunnan", "scores": null, "metadata": {"template": "_ that are also _ but not _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "animals"}} +{"query": "Mammals that are also Cetaceans excluding Miocene mammals", "docs": ["Squalodontidae", "Parasqualodon", "Blunt-snouted dolphin", "Agorophius", "Hoplocetus", "Mirocetus", "Mycteriacetus", "Waipatia", "Tagicetus", "Eurhinodelphinidae", "Otekaikea", "Orcinus citoniensis", "Hemisyntrachelus", "Ankylorhiza", "Xenorophidae", "Squalodelphinidae", "Simocetus"], "original_query": "Mammals that are also Cetaceans but not Miocene mammals", "scores": null, "metadata": {"template": "_ that are also _ but not _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "animals"}} +{"query": "what is some Endemic flora of Peninsular Malaysia that are not Trees of Malaya", "docs": ["Kochummenia", "Nepenthes macfarlanei", "Musa gracilis", "Nepenthes alba", "Schefflera pueckleri", "Prunus odorata", "Utricularia vitellina", "Musa violascens", "Barringtonia payensiana", "Begonia eiromischa", "Acrymia", "Nepenthes gracillima", "Tristaniopsis razakiana", "Begonia rajah", "Nepenthes ramispina", "Nepenthes benstonei", "Elaeocarpus cruciatus", "Rafflesia azlanii", "Dacrydium comosum", "Begonia aequilateralis"], "original_query": "Endemic flora of Peninsular Malaysia that are not Trees of Malaya", "scores": null, "metadata": {"template": "_ that are not _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "plants"}} +{"query": "Flora of the Bahamas that are Monotypic angiosperm genera", "docs": ["Acoelorrhaphe", "Piloblephis"], "original_query": "Monotypic angiosperm genera that are also Flora of the Bahamas", "scores": null, "metadata": {"template": "_ that are also _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "plants"}} +{"query": "Fish that are from both Macaronesia and the Atlantic Ocean.", "docs": ["Corkwing wrasse", "Mullus barbatus", "Polygon moray", "Atlantic trumpetfish", "Sardinella maderensis", "Garfish", "Pseudupeneus prayensis", "Gadella maraldi", "Greater weever", "Parablennius salensis"], "original_query": "Fauna of Macaronesia that are also Fish of the Atlantic Ocean", "scores": null, "metadata": {"template": "_ that are also _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "animals"}} +{"query": "2014 novels that are also Children's novels but not american", "docs": ["Girl Online", "A Song for Ella Grey", "When Everything Feels Like the Movies", "Brown Girl Dreaming", "Murder Most Unladylike", "Friday Barnes: Girl Detective", "The Whispering Skull (novel)", "Five Kingdoms: Sky Raiders", "Galunker", "Half Bad", "The Tightrope Walkers", "Cuckoo Song (novel)"], "original_query": "2014 novels that are also Children's novels but not 2014 American novels", "scores": null, "metadata": {"template": "_ that are also _ but not _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Too Ambiguous: It's too ambiguous to make a reasonable judgement. Under some likely interpretations of the two queries they have the same meaning, but under other likely interpretations they have different meanings."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Girl Online": ["Definitely relevant"], "A Song for Ella Grey": ["Definitely relevant"], "When Everything Feels Like the Movies": ["Likely relevant"], "Brown Girl Dreaming": ["Definitely relevant"], "Murder Most Unladylike": ["Definitely relevant"], "Friday Barnes: Girl Detective": ["Definitely relevant"], "The Whispering Skull (novel)": ["Likely relevant"], "Five Kingdoms: Sky Raiders": ["Likely relevant"], "Galunker": ["Likely relevant"], "Half Bad": ["Definitely relevant"], "The Tightrope Walkers": ["Definitely relevant"], "Cuckoo Song (novel)": ["Likely relevant"]}, "evidence_ratings": {"Girl Online": ["Partial"], "A Song for Ella Grey": ["Partial"], "When Everything Feels Like the Movies": ["Partial"], "Brown Girl Dreaming": ["Complete"], "Murder Most Unladylike": ["Complete"], "Friday Barnes: Girl Detective": ["Complete"], "The Whispering Skull (novel)": ["Partial"], "Five Kingdoms: Sky Raiders": ["Partial"], "Galunker": ["Partial"], "Half Bad": ["Complete"], "The Tightrope Walkers": ["Complete"], "Cuckoo Song (novel)": ["Partial"]}, "attributions": {"Girl Online": [{"2014 novels that are also Children's novels but not american": "'''''Girl Online''''' is the debut novel by English author and internet celebrity Zoe Sugg. The romance and drama novel, released on 25 November 2014 through Penguin Books, is aimed at a teen audience and focuses on a fifteen-year-old anonymous blogger and what happens when her blog goes vira"}], "A Song for Ella Grey": [{"2014 novels that are also Children's novels but not american": "'''''A Song for Ella Grey''''' is a 2014 young adult novel, written by David Almond and illustrated by Karen Radford. It is based on the legend, Orpheus and Eurydice."}], "When Everything Feels Like the Movies": [{"2014 novels that are also Children's novels": "'''''When Everything Feels Like the Movies''''' is the debut young adult novel by Raziel Reid."}], "Brown Girl Dreaming": [{"2014 novels that are also Children's novels": "''''' Brown Girl Dreaming''''' is a 2014 adolescent novel told in verse by author Jacqueline Woodson. It discusses the author's childhood as an African American growing up in the 1960s in South Carolina and New York."}], "Murder Most Unladylike": [{"2014 novels that are also Children's novels but not american": "'''''Murder Most Unladylike''''' is a 2014 children's mystery novel by British-American author Robin Stevens. It follows two schoolgirls in 1930s England solving their first murder mystery and is the first book in the 'Murder Most Unladylike' series."}], "Friday Barnes: Girl Detective": [{"2014 novels that are also Children's novels but not american": "'''''Friday Barnes, Girl Detective''''' is a 2014 Children's novel by R. A. Spratt. It is about an 11-year-old girl, Friday, who uses her detecting skills to solve mysteries at her boarding school.\n\n*2014, Australia, Random House Australia"}], "The Whispering Skull (novel)": [{"2014 novels that are also Children's novels": "'''''The Whispering Skull''''' is a young adult thriller novel by Jonathan Stroud. It is the second book in the ''Lockwood & Co.'' series. It was released on the 26 February 2015 in the UK by Corgi Children's, and released on the 16 September 2014 in the US by Disney-Hyperion."}], "Five Kingdoms: Sky Raiders": [{"2014 novels": "2014 fantasy novel"}], "Galunker": [{"014 novels that are also Children's novels but not american": "'''''Galunker''''' is a children's novel by Douglas Anthony Cooper., Illustrated by Dula Yavne"}], "Half Bad": [{"2014 novels that are also Children's novels but not american": "'''''Half Bad''''' is a 2014 young adult fantasy novel written by English debut author Sally Green."}], "The Tightrope Walkers": [{"Children's novels but not american": "It was also a 2016 Cooperative Children\u2019s Book Center Fiction for Young Adults Choice.", "2014 novels": "is a 2014 novel"}], "Cuckoo Song (novel)": [{"2014 novels": "published on 8 May 2014", "Children's novels but not american": "Macmillan in the UK"}]}}} +{"query": "2015 fiction books about racism", "docs": ["All American Boys", "The Traitor Baru Cormorant", "The Sacrifice (Oates novel)", "Early One Morning (novel)"], "original_query": "2010s books that are also both Novels about racism and 2015 fiction books", "scores": null, "metadata": {"template": "_ that are also both _ and _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"All American Boys": ["Definitely relevant"], "The Traitor Baru Cormorant": ["Likely relevant"], "The Sacrifice (Oates novel)": ["Definitely relevant"], "Early One Morning (novel)": ["Likely relevant"]}, "evidence_ratings": {"All American Boys": ["Complete"], "The Traitor Baru Cormorant": ["Complete"], "The Sacrifice (Oates novel)": ["Complete"], "Early One Morning (novel)": ["Partial"]}, "attributions": {"All American Boys": [{"2015": "The book was published in 2015 by Simon & Schuster.", "about racism": "The book follows two characters, Rashad Butler and Quinn, as they navigate racism.", "fiction books": "Amelia Elizabeth Walden Award for Young Adult Fiction"}], "The Traitor Baru Cormorant": [{"2015 fiction books": "2015 hard fantasy"}], "The Sacrifice (Oates novel)": [{"2015 fiction books about racism": "'''''The Sacrifice''''' is a 2015 novel by the American writer Joyce Carol Oates.", "racism": "Joyce Carol Oates is an American writer. Described as \"America's foremost woman of letters\" and known for her prolific output, she had written more than 40 novels before ''The Sacrifice'', some of which had dealt with issues of racism and misogyny."}], "Early One Morning (novel)": [{"2015 fiction books": "is a 2015", "about racism": "a surprisingly humorous novel, in which the characters are tenderly mocked or mock themselves\""}]}}} +{"query": "Crustaceans of the United States excluding Freshwater animals", "docs": ["Armatobalanus nefrens", "Chthamalus fragilis", "Neolithodes agassizii", "Callichirus major", "Gammarus pecos", "Callinectes sapidus", "Lithodes aequispinus", "Paguristes ulreyi", "Echinoecus pentagonus", "Paralithodes platypus", "Callinectes ornatus", "Panulirus marginatus", "Chionoecetes opilio", "Macrobrachium grandimanus", "American lobster", "Cyrtocarcinus", "Lybia edmondsoni", "Liropus minusculus", "Red king crab"], "original_query": "Crustaceans of the United States that are not Freshwater animals", "scores": null, "metadata": {"template": "_ that are not _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "animals"}} +{"query": "1960s spy comedy American thriller films", "docs": ["Arabesque (1966 film)", "Caprice (1967 film)"], "original_query": "1960s spy comedy films that are also American thriller films", "scores": null, "metadata": {"template": "_ that are also _", "domain": "films", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Arabesque (1966 film)": ["Definitely relevant"], "Caprice (1967 film)": ["Definitely relevant"]}, "evidence_ratings": {"Arabesque (1966 film)": ["Complete"], "Caprice (1967 film)": ["Partial"]}, "attributions": {"Arabesque (1966 film)": [{"1960s spy comedy American thriller films": "'''''Arabesque''''' is a 1966 American comedy thriller spy film directed by Stanley Donen and starring Gregory Peck and Sophia Loren, written by Julian Mitchell, Stanley Price, and Peter Stone based on ''The Cipher'', a 1961 novel by Alex Gordon (pseudonym of Gordon Cotler)."}], "Caprice (1967 film)": [{"1960s spy comedy American thriller films": "'''''Caprice''''' is a 1967 DeLuxe Color comedy-thriller film"}]}}} +{"query": "what are some Flora of Madagascar that are also Magnoliales genera?", "docs": ["Mauloutchia", "Fenerivia"], "original_query": "Flora of Madagascar that are also Magnoliales genera", "scores": null, "metadata": {"template": "_ that are also _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "plants"}} +{"query": "Nigerian romantic drama films excluding 2010s drama films", "docs": ["O Le Ku", "Under the Carpet", "Dear Bayo", "Silent Scandals", "Heroes & Zeros (film)", "Ovy's Voice", "This Lady Called Life", "Still Falling", "Letters to a Stranger", "Special Jollof", "Stalker (2016 film)", "Reloaded (2009 film)"], "original_query": "Nigerian romantic drama films that are not 2010s drama films", "scores": null, "metadata": {"template": "_ that are not _", "domain": "films", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"O Le Ku": ["Definitely relevant"], "Under the Carpet": ["Definitely relevant"], "Dear Bayo": ["Definitely relevant"], "Silent Scandals": ["Definitely relevant"], "Heroes & Zeros (film)": ["Likely relevant"], "Ovy's Voice": ["Definitely relevant"], "This Lady Called Life": ["Definitely relevant"], "Still Falling": ["Likely relevant"], "Letters to a Stranger": ["Definitely relevant"], "Special Jollof": ["Definitely relevant"], "Stalker (2016 film)": ["Definitely relevant"], "Reloaded (2009 film)": ["Definitely relevant"]}, "evidence_ratings": {"O Le Ku": ["Complete"], "Under the Carpet": ["Complete"], "Dear Bayo": ["Complete"], "Silent Scandals": ["Complete"], "Heroes & Zeros (film)": ["Partial"], "Ovy's Voice": ["Complete"], "This Lady Called Life": ["Complete"], "Still Falling": ["Partial"], "Letters to a Stranger": ["Complete"], "Special Jollof": ["Complete"], "Stalker (2016 film)": ["No evidence"], "Reloaded (2009 film)": ["Complete"]}, "attributions": {"O Le Ku": [{"Nigerian romantic drama films excluding 2010s drama films": "a 1997 Nigerian romantic drama film"}], "Under the Carpet": [{"Nigerian romantic drama films excluding 2010s drama films": "a 2021 Nigerian romantic drama film"}], "Dear Bayo": [{"Nigerian romantic drama films excluding 2010s drama films": "'''''Dear Bayo''''' is a 2020 Nigerian romantic drama film"}], "Silent Scandals": [{"Nigerian romantic drama films excluding 2010s drama films": "'''''Silent Scandals''''' is a 2009 Nigerian romantic drama film"}], "Heroes & Zeros (film)": [{"Nigerian": "'''''Heroes and Zeroes''''' is a 2012 Nigerian drama film written and directed by Niji Akanni; it stars Nadia Buari, Bimbo Manuel and Olu Jacobs.", "drama films excluding 2010s drama films": "'''''Heroes and Zeroes''''' is a 2012 Nigerian drama film written and directed by Niji Akanni; it stars Nadia Buari, Bimbo Manuel and Olu Jacobs."}], "Ovy's Voice": [{"Nigerian romantic drama films excluding 2010s drama films": "'''''Ovy's Voice''''' is a 2017 Nigerian romantic drama film"}], "This Lady Called Life": [{"Nigerian romantic drama films excluding 2010s drama films": "a 2020 Nigerian romance drama"}], "Still Falling": [{"Nigerian romantic drama films excluding 2010s drama films": "'''''Still Falling''''' is a 2021 Nigerian romantic drama film co-directed by Karachi Atiya and Dimbo Atiya. The film stars Daniel Etim Effiong, Sharon Ooja, Kunle Remi in the lead roles. The film had its theatrical release on 12 February 2021 on the eve of Valentine's weekend. The film was shot and set in Abuja."}], "Letters to a Stranger": [{"Nigerian romantic drama films excluding 2010s drama films": "a 2007 Nigerian romantic comedy drama film"}], "Special Jollof": [{"Nigerian romantic drama films": "''''Special Jollof''''' is a 2020 Nigerian-American romantic comedy drama film produced and directed by Emem Isong."}], "Stalker (2016 film)": [null], "Reloaded (2009 film)": [{"Nigerian romantic drama films excluding 2010s drama films": "'''''Reloaded''''' is a 2009 Nigerian romantic drama film"}]}}} +{"query": "Oceanian realm fauna that are also Fish of Southeast Asia excluding Fish of Indonesia", "docs": ["Humphead wrasse", "Clown triggerfish", "Coral grouper", "Labroides bicolor", "Areolate grouper", "Yellowmargin triggerfish", "Sufflamen bursa", "Blackbelly triggerfish", "Teardrop butterflyfish", "Hippocampus kuda", "Synchiropus splendidus", "Yellow tang", "Unicorn leatherjacket", "Leafscale gulper shark", "Milkfish", "Blackedge cusk", "Bigeye tuna", "Longnose houndshark"], "original_query": "Oceanian realm fauna that are also Fish of Southeast Asia but not Fish of Indonesia", "scores": null, "metadata": {"template": "_ that are also _ but not _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "animals"}} +{"query": "Flora of Europe and Montana", "docs": ["Carex fuliginosa", "Carex tenuiflora"], "original_query": "Flora of Europe that are also both Flora of Montana and Flora of Northern Europe", "scores": null, "metadata": {"template": "_ that are also both _ and _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "plants"}} +{"query": "Superhero comedy short films from the 1940s", "docs": ["Springman and the SS"], "original_query": "Superhero comedy films that are also 1940s short films", "scores": null, "metadata": {"template": "_ that are also _", "domain": "films", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Springman and the SS": ["Likely relevant"]}, "evidence_ratings": {"Springman and the SS": ["Partial"]}, "attributions": {"Springman and the SS": [{"short films from the 1940s": "'''''Springman and the SS''''' () is a 1946 Czechoslovak animated short film directed by Ji\u0159\u00ed Trnka featuring P\u00e9r\u00e1k, the Spring Man of Prague.", "Superhero": "Trnka's postwar interpretation of P\u00e9r\u00e1k as a quasi-superhero formed the basis for sporadic revivals of the character in Czech science fiction and comic book stories."}]}}} +{"query": "Proteaceae genera of Argentina", "docs": ["Embothrium", "Gevuina"], "original_query": "Proteaceae genera that are also Flora of Argentina", "scores": null, "metadata": {"template": "_ that are also _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "plants"}} +{"query": "Flora of Western Canada and Trees of the United States but not Northwestern United States", "docs": ["Cornus alternifolia", "Tsuga mertensiana", "Fraxinus nigra", "Acer macrophyllum", "Pinus resinosa", "Acer circinatum", "Acer glabrum", "Larix laricina", "Jack pine", "Abies amabilis", "Salix bebbiana", "Celtis occidentalis", "Abies balsamea", "Amelanchier alnifolia", "Betula neoalaskana", "Acer spicatum", "Salix amygdaloides", "Tilia americana", "Picea mariana", "Ostrya virginiana"], "original_query": "Trees of the United States that are also Flora of Western Canada but not Trees of the Northwestern United States", "scores": null, "metadata": {"template": "_ that are also _ but not _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "plants"}} +{"query": "Disaster books about London but not war", "docs": ["Wormwood (Taylor novel)", "Glamorama", "The Dead (Higson novel)", "The Book of Dave", "Left Behind (novel)", "The Enemy (Higson novel)", "The Rapture (novel)", "Good Omens", "Bloodtide (novel)", "Baptism (novel)", "Old St. Paul's (novel)", "The Snow (novel)", "Home Fire (novel)", "The World in Winter", "Fever Crumb", "Mortal Engines", "The Family Arsenal", "The Man Who Was Thursday"], "original_query": "Books about London that are also Disaster books but not Books about war", "scores": null, "metadata": {"template": "_ that are also _ but not _", "domain": "books", "fluency": ["Mostly Fluent: It has a few errors or it does not sound natural, but I can understand it."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Wormwood (Taylor novel)": ["Definitely relevant"], "Glamorama": ["Likely relevant"], "The Dead (Higson novel)": ["Definitely relevant"], "The Book of Dave": ["Definitely relevant"], "Left Behind (novel)": ["Likely relevant"], "The Enemy (Higson novel)": ["Definitely relevant"], "The Rapture (novel)": ["Likely relevant"], "Good Omens": ["Likely relevant"], "Bloodtide (novel)": ["Definitely relevant"], "Baptism (novel)": ["Definitely relevant"], "Old St. Paul's (novel)": ["Definitely relevant"], "The Snow (novel)": ["Definitely relevant"], "Home Fire (novel)": ["Likely relevant"], "The World in Winter": ["Definitely relevant"], "Fever Crumb": ["Definitely relevant"], "Mortal Engines": ["Definitely relevant"], "The Family Arsenal": ["Definitely relevant"], "The Man Who Was Thursday": ["Definitely relevant"]}, "evidence_ratings": {"Wormwood (Taylor novel)": ["Complete"], "Glamorama": ["Partial"], "The Dead (Higson novel)": ["Complete"], "The Book of Dave": ["Complete"], "Left Behind (novel)": ["Partial"], "The Enemy (Higson novel)": ["Complete"], "The Rapture (novel)": ["Partial"], "Good Omens": ["Partial"], "Bloodtide (novel)": ["Complete"], "Baptism (novel)": ["Complete"], "Old St. Paul's (novel)": ["Complete"], "The Snow (novel)": ["Complete"], "Home Fire (novel)": ["Complete"], "The World in Winter": ["Complete"], "Fever Crumb": ["Complete"], "Mortal Engines": ["Complete"], "The Family Arsenal": ["Complete"], "The Man Who Was Thursday": ["Complete"]}, "attributions": {"Wormwood (Taylor novel)": [{"Disaster books about London but not war": "The story takes place in London, where Dr. Sabian Blake is sitting in his attic at the top of his house in Bloomsbury Square, looking out to space through his telescope, in search of a special star. He is told about this star by ''The Nemorensis'', an ancient book that holds many old and powerful secrets. It has predicted that the comet Wormwood (which was foretold in the book of Revelation) is hurtling towards the earth, and would spell certain doom for London and all other lands around her."}], "Glamorama": [{"Disaster books": "The novel is a satire of modern celebrity culture; this is reflected in its premise, which features models-turned-terrorists", "Disaster books about London": "London and Paris become nothing more than a different collection of recognizable proper nouns (Notting Hill and Irvine Welsh in the first case; ''Chez Georges'' and Yves Saint Laurent in the second).\""}], "The Dead (Higson novel)": [{"Disaster books about London but not war": "'''''The Dead''''' is a novel written by Charlie Higson. The book, published by Puffin Books in the UK on 16 September 2010, is the second book in a seven-book series, titled The Enemy. ''The Dead'' takes place in London, a year before the events in the previous book (''The Enemy'', released in the UK by Puffin Books on 3 September 2009), two weeks after a worldwide sickness has infected adults turning them into something related to voracious, cannibalistic zombies."}], "The Book of Dave": [{"Disaster books about London but not war": "''The Book of Dave'' tells the story of an angry and mentally ill London taxi driver named Dave Rudman, who writes and has printed on metal a book of his rantings against women and thoughts on custody rights for fathers. These stem from his anger with his ex-wife, Michelle, who he believes is unfairly keeping him from his son. Equally influential in Dave's book is The Knowledge\u2014the intimate familiarity with the city of London required of its cabbies."}], "Left Behind (novel)": [{"about London": "on board a plane piloted by Rayford Steele en route to London."}], "The Enemy (Higson novel)": [{"Disaster books about London": "''''The Enemy''''' is a post-apocalyptic young adult horror novel written by Charlie Higson. The book takes place in London, United Kingdom, after a worldwide sickness has infected adults, turning them into something akin to voracious, cannibalistic zombies."}], "The Rapture (novel)": [{"Disaster books about London but not war": "'''''The Rapture: In the Twinkling of an Eye/Countdown to the Earth's Last Days''''' is the 3rd prequel novel in the ''Left Behind'' series, written by Tim LaHaye and Jerry B. Jenkins in 2006. This book is the final of the three prequels and covers events leading up to the first book ''Left Behind''. The narrative of the novel ''The Rapture'' includes events that take place during the first chapters of ''Left Behind'' and provides a backdrop story for the book ''Left Behind''. The book was released on Tuesday, June 6, 2006 (6/6/6), which is the Number of the Beast, a concept that plays a large part later in the series. It takes place from 14 months before to the day of the Rapture.\nThe stage is set as Nicolae Carpathia ruthlessly eliminates any obstacles in his rise to power. Pan-Con Airlines Captain Rayford Steele prepares for a flight to London with beautiful flight attendant Hattie Durham. Because of his wife's newfound faith, Rayford looks forward to time\u2014and the possibilities\u2014with Hattie."}], "Good Omens": [{"Disaster books about London but not war": "'''''Good Omens: The Nice and Accurate Prophecies of Agnes Nutter, Witch''''' is a 1990 novel written as a collaboration between the English authors Terry Pratchett and Neil Gaiman."}], "Bloodtide (novel)": [{"about London but not war": "The book deals with two warring gangs, named the Volsons and the Connors. Each controls a large portion of London, since the city was blockaded from the outside world. Val, head of the Volsons, wishes for peace between the two and offers his daughter Signy as Conor's wife in order to broker a treaty. His hopes are to unite London and destroy the half-men who occupy the outer rim of London.", "Disaster books": "This story takes place in the future, where London is a wasteland where two clans war."}], "Baptism (novel)": [{"Disaster books about London": "Baptism''''' is a 2012 crime novel by British author Max Kinnings"}], "Old St. Paul's (novel)": [{"Disaster books about London but not war": "It is a historical romance that describes the events of the Great Plague of London and the Great Fire of London."}], "The Snow (novel)": [{"Disaster books about London but not war": "'''''The Snow''''', published in 2004, is a science fiction novel by British writer Adam Roberts"}], "Home Fire (novel)": [{"Disaster books about London": "Shamsie began ''Home Fire'' at the suggestion of London theatre director Jatinder Verma that Shamsie write a modern update of ''Antigone'', by Sophocles."}], "The World in Winter": [{"Disaster books about London but not war": "'''''The World in Winter''''' (US title '''''The Long Winter''''') is a 1962 post-apocalyptic science fiction novel by British writer John Christopher. It deals with a new ice age caused by a reduction in the output of the Sun.\n\n\nThe story involves a new ice age hitting Europe, British refugees fleeing to Nigeria, and what a later group find when they return.\n\n\nAs the story opens, Andrew Leedon, a London-based television documentary producer, is given a new story to research: an Italian scientist, Fratellini, has proposed an imminent fall in solar radiation for the forthcoming few years which may lead to harsher winters. Leedon meets with David Cartwell, a Home Office civil servant and useful source, to see if he can find out more. Cartwell quickly becomes a close friend of Leedon, but also begins an affair with Leedon's wife, Carol."}], "Fever Crumb": [{"Disaster books about London but not war": "The Scriven were a race superior to humans and were violent rulers of London until rebellion forced them out."}], "Mortal Engines": [{"Disaster books about London but not war": "'''''Mortal Engines''''' is a young-adult science fantasy novel by Philip Reeve, published by Scholastic UK in 2001. The book focuses on a futuristic, steampunk version of London, now a giant machine striving to survive on a world that is running out of resources."}], "The Family Arsenal": [{"Disaster books about London but not war": "'''''The Family Arsenal''''' is a novel by Paul Theroux originally published in 1976. It is a political thriller following the acts of a terrorist cell in London."}], "The Man Who Was Thursday": [{"Disaster books about London but not war": "'The Man Who Was Thursday: A Nightmare''''' is a 1908 novel by G. K. Chesterton."}]}}} +{"query": "Plants Afghanistan and Israel have in common", "docs": ["Phalaris minor", "Carduus pycnocephalus", "Parapholis incurva", "Brassica nigra"], "original_query": "Flora of Afghanistan that are also Flora of Israel", "scores": null, "metadata": {"template": "_ that are also _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "plants"}} +{"query": "what are birds of South China that are also Birds of Vietnam but not Birds of Myanmar?", "docs": ["Kloss's leaf warbler", "Yellow-billed nuthatch", "Slaty-blue flycatcher", "Fork-tailed sunbird", "Greyish-crowned leafbird", "Blue-rumped pitta", "White-bellied green pigeon", "Ratchet-tailed treepie", "Bar-backed partridge", "Tawny fish owl", "Streaked barwing", "White-eared night heron", "Scaly-breasted cupwing", "Yellow-browed tit", "White-shouldered starling", "Black-headed sibia", "White-winged magpie", "Indochinese green magpie", "Streak-breasted scimitar babbler"], "original_query": "Fauna of South China that are also Birds of Vietnam but not Birds of Myanmar", "scores": null, "metadata": {"template": "_ that are also _ but not _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "animals"}} +{"query": "Afrotropical flora of Sierra Leone but not Nigeria", "docs": ["Aphanocalyx microphyllus subsp. compactus", "Monodora crispata", "Oxyanthus tubiflorus", "Strephonema pseudocola", "Phlegmariurus phlegmaria", "Alsophila camerooniana", "Cyperus sphacelatus", "Utricularia macrocheilos", "Gilbertiodendron splendidum", "Xysmalobium samoritourei", "Utricularia spiralis", "Ancistrochilus rothschildianus", "Droogmansia scaettaiana", "Utricularia tetraloba", "Elaeis", "Droogmansia chevalieri", "Neostenanthera hamata", "Cyperus incompressus", "Cyperus congensis", "Scleria robinsoniana"], "original_query": "Afrotropical realm flora that are also Flora of Sierra Leone but not Flora of Nigeria", "scores": null, "metadata": {"template": "_ that are also _ but not _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "plants"}} +{"query": "Trees that the Pacific, Australia, and the Ryukyu Islands have in common", "docs": ["Nypa fruticans", "Sonneratia alba"], "original_query": "Trees of the Pacific that are also both Trees of Australia and Flora of the Ryukyu Islands", "scores": null, "metadata": {"template": "_ that are also both _ and _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "plants"}} +{"query": "Carnivorous plants that are from Liberia and Paleotropical", "docs": ["Utricularia striatula", "Utricularia simulans", "Utricularia stanfieldii", "Utricularia spiralis", "Utricularia andongensis", "Utricularia firmula", "Utricularia tortilis", "Utricularia pubescens"], "original_query": "Carnivorous plants that are also both Flora of Liberia and Paleotropical flora", "scores": null, "metadata": {"template": "_ that are also both _ and _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "plants"}} +{"query": "Peninsular Malaysia trees that are also Trees of Indo-China but not Flora of Borneo", "docs": ["Shorea singkawang", "Shorea hypochra", "Vatica diospyroides", "Ficus subpisocarpa", "Merrillia", "Borassodendron machadonis", "Hopea pierrei", "Vatica stapfiana", "Shorea roxburghii", "Illicium tenuifolium", "Magnolia praecalva", "Hopea helferi", "Vatica harmandiana", "Haldina", "Fordia pauciflora", "Shorea henryana", "Butea monosperma", "Dipterocarpus dyeri"], "original_query": "Trees of Peninsular Malaysia that are also Trees of Indo-China but not Flora of Borneo", "scores": null, "metadata": {"template": "_ that are also _ but not _", "relevance_ratings": null, "evidence_ratings": null, "attributions": null, "domain": "plants"}} +{"query": "Crime books that about the arts but not about film", "docs": ["The D Case", "Unfinished Portrait (novel)", "Death of a Ghost", "The Five Red Herrings", "Wycliffe and the Winsor Blue", "Artists in Crime", "The Documents in the Case", "Killing Critics", "Strangers on a Train (novel)", "The Shame of the Cities", "The Blind Man of Seville", "The Tattooed Potato and Other Clues"], "original_query": "Crime books that are also Books about the arts but not Books about film", "scores": null, "metadata": {"template": "_ that are also _ but not _", "domain": "books", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"The D Case": ["Likely relevant"], "Unfinished Portrait (novel)": ["Likely relevant"], "Death of a Ghost": ["Likely relevant"], "The Five Red Herrings": ["Definitely relevant"], "Wycliffe and the Winsor Blue": ["Definitely relevant"], "Artists in Crime": ["Likely relevant"], "The Documents in the Case": ["Definitely relevant"], "Killing Critics": ["Definitely relevant"], "Strangers on a Train (novel)": ["Likely relevant"], "The Shame of the Cities": ["Definitely relevant", "Definitely not relevant"], "The Blind Man of Seville": ["Likely relevant"], "The Tattooed Potato and Other Clues": ["Likely relevant"]}, "evidence_ratings": {"The D Case": ["Partial"], "Unfinished Portrait (novel)": ["Partial"], "Death of a Ghost": ["Partial"], "The Five Red Herrings": ["Complete"], "Wycliffe and the Winsor Blue": ["Complete"], "Artists in Crime": ["Partial"], "The Documents in the Case": ["Complete"], "Killing Critics": ["Complete"], "Strangers on a Train (novel)": ["Partial"], "The Shame of the Cities": ["Complete", "No evidence"], "The Blind Man of Seville": ["Complete"], "The Tattooed Potato and Other Clues": ["Partial"]}, "attributions": {"The D Case": [{"about the arts": "This is established under the setting of a convention that has been organized by affluent Japanese patrons to finish unfinished works of art"}], "Unfinished Portrait (novel)": [{"about the arts but not about film": "But, while on an exotic island, Celia meets Larraby, a successful portrait painter, who spends a night talking with her, and learning her deepest fears."}], "Death of a Ghost": [{"Crime books": "'''''Death of a Ghost''''' is a crime novel by Margery Allingham,"}], "The Five Red Herrings": [{"Crime books that about the arts but not about film": "Sandy Campbell is a talented painter, but also a notoriously quarrelsome drunkard. When he is found dead in a stream, with a still-wet half-finished painting on the bank above, it is assumed that he fell in accidentally, fracturing his skull. Lord Peter Wimsey, who is in the region on a fishing holiday, suspects murder when he realises that something is missing from the scene which makes it impossible for Campbell to have worked on the painting."}], "Wycliffe and the Winsor Blue": [{"Crime books that about the arts but not about film": "''''Wycliffe and the Winsor Blue''''' (1987) is a crime novel by Cornish writer W. J. Burley."}], "Artists in Crime": [null], "The Documents in the Case": [{"Crime books that about the arts but not about film": "'''''The Documents in the Case''''' is a 1930 novel by Dorothy L. Sayers and Robert Eustace. It is the only one of Sayers's twelve major crime novels not to feature Lord Peter Wimsey, her most famous detective character. However, the forensic analyst Sir James Lubbock, who appears or is mentioned in several of the Wimsey novels, also appears in ''The Documents in the Case''.\nThis is an epistolary novel, told primarily in the form of letters between some of the characters, using the multiple narrative technique associated with Modernist novelists of the period. This collection of documents\u2014hence the novel's title\u2014is explained as a dossier of evidence collected by the victim's son as part of his campaign to obtain justice for his father."}], "Killing Critics": [{"Crime books that about the arts but not about film": "'''Killing Critics''' is the third book in the Kathleen Mallory series written by Carol O'Connell.\nMallory investigates the murder of Dean Starr, an artist killed in the middle of an exhibition. The killer made the murder appear to be performance art."}], "Strangers on a Train (novel)": [{"Crime books that about the arts but not about film": "'''''Strangers on a Train''''' (1950) is a psychological thriller novel by Patricia Highsmith about two men whose lives become entangled after one of them proposes they \"trade\" murders."}], "The Shame of the Cities": [{"Crime books that about the arts but not about film": "It reports on the workings of corrupt political machines in several major U.S. cities, along with a few efforts to combat them. It is considered one of several early major pieces of muckraking journalism, though Steffens later claimed that this work made him \"the first muckraker.\""}, {"Crime books that about the arts": "It reports on the workings of corrupt political machines in several major U.S. cities, along with a few efforts to combat them."}], "The Blind Man of Seville": [{"Crime books": "'''''The Blind Man of Seville''''' is a 2003 crime novel and thriller by British writer Robert Wilson.", "that about the arts": "The investigation into the restaurateur's turbulent life sends Falc\u00f3n trawling through his own past and the ferociously candid journals of his late father, a world-famous artist."}], "The Tattooed Potato and Other Clues": [{"Crime books that about the arts": "an art-school student in Greenwich Village, answers an ad for a job as a painter's assistant at Number 12 Cobble Lane. The painter, Garson, evaluates and hires her; in her duties of cleaning paintbrushes and answering the door, she becomes involved in Garson's mysterious affairs, as well of those of his downstairs neighbors,"}]}}} +{"query": "Documentary films about education and intellectual disability", "docs": ["Educating Peter", "Graduating Peter"], "original_query": "Documentary films about education that are also Films about intellectual disability", "scores": null, "metadata": {"template": "_ that are also _", "domain": "films", "fluency": ["Fluent: It is clear, and grammatically correct."], "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], "relevance_ratings": {"Educating Peter": ["Definitely relevant"], "Graduating Peter": ["Definitely relevant"]}, "evidence_ratings": {"Educating Peter": ["Complete"], "Graduating Peter": ["Complete"]}, "attributions": {"Educating Peter": [{"Documentary films about education and intellectual disability": "'''''Educating Peter''''' is a 1992 American short documentary film directed by Gerardine Wurzburg about Peter Gwazdauskas, a special needs student with Down syndrome, and his inclusion in a standard third grade classroom in Blacksburg, Virginia. It won an Oscar at the 65th Academy Awards in 1993 for Documentary Short Subject."}], "Graduating Peter": [{"Documentary films about education and intellectual disability": "'''''Graduating Peter''''' is a 2001 American documentary film directed by Gerardine Wurzburg. This is a sequel to the 1992 Academy Award-winning short documentary film ''Educating Peter''; where it follows the continuing adventures of Peter Gwazdauskas, a special needs student with Down syndrome, in his middle school and high school life as well as his high school graduation. The documentary was filmed in Blacksburg, Virginia."}]}}} diff --git a/baseline_quest/decompose_analyze_varying_k.py b/baseline_quest/decompose_analyze_varying_k.py new file mode 100644 index 0000000..24cf4be --- /dev/null +++ b/baseline_quest/decompose_analyze_varying_k.py @@ -0,0 +1,92 @@ +import os +import csv +import argparse +from typing import List, Set + +from lib.chroma_utils import read_jsonl + +def calculate_recall(gold_docs: Set[str], predicted_docs: List[str]) -> float: + if not gold_docs: + return 1.0 + + pred_set = set(predicted_docs) + intersection = gold_docs.intersection(pred_set) + return len(intersection) / len(gold_docs) + +def run_analysis(gold_path: str, pred_path: str, output_path: str, query_index: int): + # 2. Load Gold Data (using chroma_utils) + gold_examples = list(read_jsonl(gold_path)) + + # 3. Extract the target Query + target_gold = gold_examples[query_index] + gold_docs_set = set(target_gold.get("docs", [])) + query_text = target_gold.get("query", "Unknown Query") + + pred_examples = list(read_jsonl(pred_path)) + + os.makedirs(os.path.dirname(output_path), exist_ok=True) + + with open(output_path, "w", newline="", encoding="utf-8") as f_csv: + csv_writer = csv.writer(f_csv) + # CSV Header + csv_writer.writerow(["query", "k1", "k2", "recall_docs_1", "recall_docs_2", "recall_final"]) + + stats = { + "total_recall_1": 0.0, + "total_recall_2": 0.0, + "total_recall_final": 0.0, + "count": 0 + } + + for pred in pred_examples: + # Extract parameters + k1 = pred.get("k1") + k2 = pred.get("k2") + + if k1 is None or k2 is None: + continue + + docs_1 = pred.get("docs_1", []) + docs_2 = pred.get("docs_2", []) + docs_final = pred.get("docs", []) + + # Calculate Recall + r1 = calculate_recall(gold_docs_set, docs_1) + r2 = calculate_recall(gold_docs_set, docs_2) + r_final = calculate_recall(gold_docs_set, docs_final) + + # Write row + csv_writer.writerow([ + query_text, + k1, + k2, + f"{r1:.6f}", + f"{r2:.6f}", + f"{r_final:.6f}" + ]) + + # Accumulate stats + stats["total_recall_1"] += r1 + stats["total_recall_2"] += r2 + stats["total_recall_final"] += r_final + stats["count"] += 1 + +def main(): + parser = argparse.ArgumentParser(description="Analyze Recall for varying k parameters.") + + # Default args match your file structure + parser.add_argument("--gold", type=str, default="../../train_subset.jsonl", + help="Path to gold standard JSONL.") + parser.add_argument("--pred", type=str, default="pred_query_10_varying_k_modified.jsonl", + help="Path to predictions JSONL.") + parser.add_argument("--out", type=str, default="query_10_varying_k_results_modified.csv", + help="Output CSV path.") + parser.add_argument("--index", type=int, default=10, + help="Index of the query in the gold file (0-based). Default 10.") + + args = parser.parse_args() + + run_analysis(args.gold, args.pred, args.out, args.index) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/baseline_quest/decompose_retrieve.py b/baseline_quest/decompose_retrieve.py new file mode 100644 index 0000000..4be739b --- /dev/null +++ b/baseline_quest/decompose_retrieve.py @@ -0,0 +1,104 @@ +import os +import sys +import yaml +import logging +import argparse + +# SQLite compatibility +try: + __import__('pysqlite3') + sys.modules['sqlite3'] = sys.modules.pop('pysqlite3') +except ImportError: + pass + +# Configure logging +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +# Import from lib +from lib.decomposition_generator import generate_strategies +from lib.decomposition_executor import execute_all_strategies +from lib.retrieval_analyzer import analyze_results + +CONFIG_PATH = "config.yaml" + +def load_config(path: str): + if not os.path.exists(path): + raise FileNotFoundError(f"Config file not found: {path}") + with open(path, 'r') as f: + return yaml.safe_load(f) + +def main(): + parser = argparse.ArgumentParser(description="Run Decomposition Pipeline") + parser.add_argument( + "--mode", + # Removed "auto" from choices + choices=["generate", "execute", "analyze"], + default="generate", + help=( + "generate: Create strategy files only (then stop for review). " + "execute: Run existing strategy files and analyze results. " + "analyze: Run analysis on existing predictions only." + ) + ) + args = parser.parse_args() + + logger.info(f"Loading configuration from {CONFIG_PATH}...") + config = load_config(CONFIG_PATH) + + queries_path = config['data']['queries_file'] + subset_name = os.path.splitext(os.path.basename(queries_path))[0] + target_k = config['decomposition'].get('top_k', 100) + + strategies_dir = f"results/decomposition/{subset_name}_k{target_k}" + pred_output_path = f"results/decomposition/pred_set_ops_{subset_name}.jsonl" + report_output_path = f"results/decomposition/results/recall_report_{subset_name}.txt" + + # MODE: GENERATE + if args.mode == "generate": + logger.info("=== Step 1: Generating Decompositions ===") + generate_strategies( + queries_path=queries_path, + output_dir=strategies_dir, + target_k=target_k, + llm_model=config.get('decomposition', {}).get('llm_model', 'gpt-4o-mini') + ) + + logger.info("\n" + "="*60) + logger.info(f"GENERATION COMPLETE. Strategies saved to: {strategies_dir}") + logger.info("IMPORTANT: Please review/edit the generated Python files.") + logger.info("When ready, run this script again with: --mode execute") + logger.info("="*60 + "\n") + return # Stop execution after generation + + # MODE: EXECUTE + elif args.mode == "execute": + logger.info("=== Step 2: Executing Strategies ===") + + if not os.path.exists(strategies_dir) or not os.listdir(strategies_dir): + logger.error(f"No strategies found in {strategies_dir}. Run with --mode generate first.") + return + + execute_all_strategies( + strategies_dir=strategies_dir, + queries_path=queries_path, + output_path=pred_output_path, + config=config + ) + # Fall through to analysis (Step 3) + + if args.mode in ["execute", "analyze"]: + logger.info("=== Step 3: Analyzing Results ===") + if os.path.exists(pred_output_path): + analyze_results( + gold_path=queries_path, + pred_path=pred_output_path, + output_report_path=report_output_path + ) + else: + logger.error(f"Prediction file not found at {pred_output_path}. Cannot analyze.") + + logger.info("Decomposition pipeline tasks completed.") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/baseline_quest/examples/example_query.jsonl b/baseline_quest/examples/example_query.jsonl new file mode 100644 index 0000000..60f43d3 --- /dev/null +++ b/baseline_quest/examples/example_query.jsonl @@ -0,0 +1,61 @@ +{ + "query": "Non Horror demon novels.", + "docs": ["List of the Lost", "Blood Price", "The Black Spider", "The Castle in the Forest", "The Devil in Love (novel)", "Melmoth the Wanderer", "Practical Demonkeeping", "Artemis Fowl and the Lost Colony", "The Black Tattoo", "Good Omens", "Eric (novel)"], "original_query": "Demon novels that are not Horror novel series", + "scores": null, + "metadata": { + "template": "_ that are not _", + "domain": "books", + "fluency": ["Fluent: It is clear, and grammatically correct."], + "meaning": ["Same Meaning: The paraphrased query asks for the same set of items as the original query. All the highlighted clauses are included."], "naturalness": ["Yes - A user could plausibly issue this query."], + "relevance_ratings": { + "List of the Lost": ["Definitely relevant"], + "Blood Price": ["Definitely relevant"], + "The Black Spider": ["Likely relevant"], + "The Castle in the Forest": ["Likely relevant"], + "The Devil in Love (novel)": ["Definitely relevant"], + "Melmoth the Wanderer": ["Definitely relevant"], + "Practical Demonkeeping": ["Likely relevant"], + "Artemis Fowl and the Lost Colony": ["Definitely relevant"], + "The Black Tattoo": ["Definitely relevant"], + "Good Omens": ["Likely relevant"], + "Eric (novel)": ["Definitely relevant"]}, + "evidence_ratings": {"List of the Lost": ["Complete"], + "Blood Price": ["Complete"], + "The Black Spider": ["Partial"], + "The Castle in the Forest": ["Partial"], + "The Devil in Love (novel)": ["Complete"], + "Melmoth the Wanderer": ["Complete"], + "Practical Demonkeeping": ["Partial"], + "Artemis Fowl and the Lost Colony": ["Complete"], + "The Black Tattoo": ["Complete"], + "Good Omens": ["Partial"], + "Eric (novel)": ["Complete"] + }, + "attributions": { + "List of the Lost": [{"Non Horror demon novels.": "The book is about a 1970s relay team in Boston who accidentally kill a homeless person, whose death brings misfortune to the team."}], + "Blood Price": [{"Non Horror demon novels.": "He tells her that the killer is a demon, that she actually did see him disappear."}], + "The Black Spider": [{"demon": "The hunter used his demonic powers to instill a curse in the kiss, which would ensure his payment."}], + "The Castle in the Forest": [{"Non Horror demon novels.": "'''''The Castle in the Forest''''' is the last novel by writer Norman Mailer, published in the year of his death, 2007. It is the story of Adolf Hitler's childhood as seen through the eyes of Dieter, a demon sent to put him on his destructive path. The novel explores the idea that Hitler was the product of incest. It forms a thematic contrast with the writer's immediately previous novel ''The Gospel According to the Son'' (1999), which deals with the early life of Jesus. It received a good deal of praise, including a glowing review from Lee Siegel of ''The New York Times Book Review'', and was the ''New York Times'' Bestseller for 2007."}], + "The Devil in Love (novel)": [{"Non Horror demon novels.": "Author of ''The Devil in Love, Jacques Cazotte''\n'''''The Devil in Love''''' (, 1772) is an occult romance by Jacques Cazotte which tells of a demon, or devil, who falls in love with a young Spanish nobleman named Don Alvaro, an amateur human dabbler, and attempts, in the guise of a young woman, to win his affections."}], + "Melmoth the Wanderer": [{"Non Horror demon novels.": "'''''Melmoth the Wanderer''''' is an 1820 Gothic novel by Irish playwright, novelist and clergyman Charles Maturin. The novel's titular character is a scholar who sold his soul to the devil in exchange for 150 extra years of life, and searches the world for someone who will take over the pact for him, in a manner reminiscent of the Wandering Jew."}], + "Practical Demonkeeping": [{"Non Horror demon novels": "His first novel, it deals with a demon from Hell and his master."}], + "Artemis Fowl and the Lost Colony": [{"Non Horror demon novels.": "In Barcelona, Spain, Artemis Fowl II and Butler, his bodyguard, wait for a demon. They suddenly encounter a demon who transports Artemis through time."}], + "The Black Tattoo": [{"Non Horror demon novels.": "'''''The Black Tattoo''''' is a young adult fantasy novel by Sam Enthoven, published in 2006. It deals with a boy, Charlie, becoming possessed by a demon that manifests itself in the form of a black tattoo on his body."}], + "Good Omens": [{"demon novels.": "There are attempts by the angel Aziraphale and the demon Crowley to sabotage the coming of the end times, having grown accustomed to their comfortable surroundings in England."}], + "Eric (novel)": [{"demon novels.": "the Demon King"}] + } + } +} + +{ + "query": "what are Oceanian realm fauna that are also both Birds of North America and Fauna of Europe", "docs": ["Sooty tern", "Bulwer's petrel", "Black noddy", "Bar-tailed godwit", "Masked booby", "Red-footed booby", "Roseate tern"], + "original_query": "Oceanian realm fauna that are also both Birds of North America and Fauna of Europe", + "scores": null, + "metadata": { + "template": "_ that are also both _ and _", + "relevance_ratings": null, + "evidence_ratings": null, + "attributions": null, + "domain": "animals" + } +} diff --git a/baseline_quest/examples/query_10.py b/baseline_quest/examples/query_10.py new file mode 100644 index 0000000..f9e4e9b --- /dev/null +++ b/baseline_quest/examples/query_10.py @@ -0,0 +1,18 @@ +from typing import Dict + +# Strategy for Query 10: 1912 films set in England (No Truncation) +def execute_query(retrieve, k1, k2): + films_1912_dict = retrieve("1912 films", k1) + films_set_in_england_dict = retrieve("English films", k2) + + films_1912_titles = set(films_1912_dict.keys()) + films_set_in_england_titles = set(films_set_in_england_dict.keys()) + + intersecting_titles = films_1912_titles & films_set_in_england_titles + + final_docs_dict = { + title: films_1912_dict[title] + for title in intersecting_titles + } + + return final_docs_dict, films_1912_dict, films_set_in_england_dict \ No newline at end of file diff --git a/baseline_quest/examples/query_9.py b/baseline_quest/examples/query_9.py new file mode 100644 index 0000000..9c976eb --- /dev/null +++ b/baseline_quest/examples/query_9.py @@ -0,0 +1,13 @@ +# Strategy for Query 9: romance films from New Zealand (No Truncation) +def execute_query(retrieve, k1, k2): + # Step 1: Retrieve all romance films + romance_films = retrieve("romance films", k1) + + # Step 2: Retrieve all films from New Zealand + new_zealand_films = retrieve("films from New Zealand", k2) + + # Step 3: Intersect the two sets to find romance films specifically from New Zealand + romance_nz_films = romance_films & new_zealand_films + + # Returning the result as a list + return list(romance_nz_films) diff --git a/baseline_quest/index_documents.py b/baseline_quest/index_documents.py new file mode 100644 index 0000000..47e1039 --- /dev/null +++ b/baseline_quest/index_documents.py @@ -0,0 +1,149 @@ +__import__('pysqlite3') +import sys +sys.modules['sqlite3'] = sys.modules.pop('pysqlite3') + +import os +import signal +import logging +import yaml +from tqdm import tqdm +from transformers import AutoTokenizer + +# Import tools from the library +from lib.chroma_utils import get_db_collection, read_jsonl, stable_entity_id, chunk_by_tokens + +# Setup Logging +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(levelname)s - %(message)s", + handlers=[ + logging.FileHandler("indexing.log"), + logging.StreamHandler(sys.stdout) + ] +) +logger = logging.getLogger(__name__) + +def upsert_in_batches(collection, ids, documents, metadatas, batch_size): + for i in range(0, len(ids), batch_size): + j = i + batch_size + collection.upsert( + ids=ids[i:j], + documents=documents[i:j], + metadatas=metadatas[i:j] + ) + +def main(): + # 1. Load Config + config_path = "config.yaml" + if not os.path.exists(config_path): + logger.error("config.yaml not found.") + sys.exit(1) + + with open(config_path, "r") as f: + config = yaml.safe_load(f) + + idx_conf = config['indexing'] + doc_path = config['data']['document_path'] + + logger.info(f"Source file: {doc_path}") + logger.info(f"Collection: {idx_conf['chroma']['collection']}") + logger.info(f"Strategy: {'First 512 Only' if idx_conf['index_first_512'] else 'Full Sliding Window'}") + + # 2. Initialize Resources + if not os.path.isfile(doc_path): + logger.error(f"Data file not found: {doc_path}") + sys.exit(1) + + collection = get_db_collection(config, clear_existing=True) + tokenizer = AutoTokenizer.from_pretrained(idx_conf['embedding_model'], use_fast=True) + + # 3. Prepare Loop + total_docs = sum(1 for _ in read_jsonl(doc_path)) + docs_iter = read_jsonl(doc_path) + + p_docs = tqdm(total=total_docs, desc="Docs", unit="doc") + p_units = tqdm(total=0, desc="Chunks", unit="chunk") + + batch_ids, batch_docs, batch_metas = [], [], [] + running_total = 0 + + interrupted = False + + def handle_sigint(sig, frame): + nonlocal interrupted + interrupted = True + logger.warning("Interrupt received. Finishing current batch...") + + original_sigint = signal.signal(signal.SIGINT, handle_sigint) + + # 4. Processing Loop + try: + for raw in docs_iter: + if interrupted: break + + title = (raw.get("title") or "").strip() or "untitled" + text = (raw.get("text") or "").strip() + entity_id = stable_entity_id(title, text) + + if idx_conf['index_first_512']: + toks = tokenizer.encode(text, add_special_tokens=False) + truncated = toks[:idx_conf['chunk_size']] + chunk_text = tokenizer.decode(truncated, skip_special_tokens=True).strip() + if not chunk_text: chunk_text = title + + batch_ids.append(entity_id) + batch_docs.append(chunk_text) + batch_metas.append({ + "entity_id": entity_id, + "title": title, + "chunk_index": 0, + "n_chunks": 1, + "source": os.path.basename(doc_path), + }) + running_total += 1 + + else: + chunks = chunk_by_tokens( + text, tokenizer, + idx_conf['chunk_size'], + idx_conf['overlap'] + ) + if not chunks: chunks = [title] + + n_chunks = len(chunks) + for idx, ch in enumerate(chunks): + # Create unique ID for chunk + cid = f"{entity_id}__{idx:04d}" + batch_ids.append(cid) + batch_docs.append(ch) + batch_metas.append({ + "entity_id": entity_id, + "title": title, + "chunk_index": idx, + "n_chunks": n_chunks, + "source": os.path.basename(doc_path), + }) + running_total += n_chunks + + p_docs.update(1) + p_units.total = running_total + p_units.refresh() + + if len(batch_ids) >= idx_conf['batch_size']: + upsert_in_batches(collection, batch_ids, batch_docs, batch_metas, idx_conf['batch_size']) + p_units.update(len(batch_ids)) + batch_ids.clear(); batch_docs.clear(); batch_metas.clear() + + # Final Flush + if batch_ids: + upsert_in_batches(collection, batch_ids, batch_docs, batch_metas, idx_conf['batch_size']) + p_units.update(len(batch_ids)) + + finally: + signal.signal(signal.SIGINT, original_sigint) + p_docs.close() + p_units.close() + logger.info(f"Indexing complete. Total units: {p_units.n}") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/baseline_quest/lib/__init__.py b/baseline_quest/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/baseline_quest/lib/chroma_utils.py b/baseline_quest/lib/chroma_utils.py new file mode 100644 index 0000000..f6d8eb7 --- /dev/null +++ b/baseline_quest/lib/chroma_utils.py @@ -0,0 +1,111 @@ +import os +import re +import json +import hashlib +import unicodedata +import chromadb +import logging +from typing import List, Dict, Iterable +from unidecode import unidecode +from sentence_transformers import SentenceTransformer +from transformers import AutoTokenizer + +logger = logging.getLogger(__name__) + +# --- Text Processing Utils --- + +def normalize_title_slug(s: str) -> str: + if not s: + return "untitled" + t = unicodedata.normalize("NFC", s).strip() + t = unidecode(t) + t = re.sub(r"\s+", " ", t) + t = re.sub(r"[^A-Za-z0-9 _\-.]", "", t).strip().replace(" ", "_") + return t or "untitled" + +def stable_entity_id(title: str, text: str) -> str: + slug = normalize_title_slug(title) + h = hashlib.sha1((title + "\n" + (text or "")).encode("utf-8")).hexdigest()[:8] + return f"{slug}-{h}" + +def read_jsonl(path: str) -> Iterable[Dict]: + with open(path, "r", encoding="utf-8") as f: + for idx, line in enumerate(f, 1): + line = line.strip() + if not line: + continue + try: + yield json.loads(line) + except json.JSONDecodeError as e: + logger.warning(f"Skipping malformed JSON on line {idx}: {e}") + +def chunk_by_tokens(text: str, tokenizer, chunk_tokens: int, overlap_tokens: int) -> List[str]: + toks = tokenizer.encode(text, add_special_tokens=False) + if not toks: + return [] + chunks = [] + step = max(1, chunk_tokens - overlap_tokens) + for start in range(0, len(toks), step): + end = min(start + chunk_tokens, len(toks)) + sub = toks[start:end] + if not sub: + break + chunk_text = tokenizer.decode(sub, skip_special_tokens=True).strip() + if chunk_text: + chunks.append(chunk_text) + if end >= len(toks): + break + return chunks + +# --- ChromaDB & Embedding Utils --- + +class STEmbeddingFn: + def __init__(self, model_name: str, device: str = None, batch_size: int = 64): + self.model_name = model_name + self.model = SentenceTransformer(model_name, device=device) + self.batch_size = batch_size + + def __call__(self, input: List[str]) -> List[List[float]]: + if not input: + return [] + embs = self.model.encode( + input, + batch_size=self.batch_size, + convert_to_numpy=True, + normalize_embeddings=True, + show_progress_bar=False, + ) + return embs.tolist() + + def name(self) -> str: + return f"sentence-transformers:{self.model_name}" + +def get_db_collection(config: Dict, clear_existing: bool = False): + """ + Initializes ChromaDB client and returns the collection based on config. + """ + # Extract config values + persist_dir = config['indexing']['chroma']['persist_dir'] + collection_name = config['indexing']['chroma']['collection'] + model_name = config['indexing']['embedding_model'] + + # Ensure directory exists + os.makedirs(persist_dir, exist_ok=True) + + # Connect + client = chromadb.PersistentClient(path=persist_dir) + embed_fn = STEmbeddingFn(model_name=model_name) + + if clear_existing: + try: + client.delete_collection(collection_name) + logger.info(f"Deleted existing collection '{collection_name}'") + except Exception: + pass + + collection = client.get_or_create_collection( + name=collection_name, + embedding_function=embed_fn, + ) + + return collection \ No newline at end of file diff --git a/baseline_quest/lib/decomposition_executor.py b/baseline_quest/lib/decomposition_executor.py new file mode 100644 index 0000000..6d5f36d --- /dev/null +++ b/baseline_quest/lib/decomposition_executor.py @@ -0,0 +1,104 @@ +import importlib.util +import json +import os +import sys +from tqdm import tqdm +from functools import partial + +from chroma_utils import get_db_collection +from retriever import retrieve + +def load_strategy_functions(strategy_dir): + """ + Import all query_*.py files in the given folder. + Returns a dictionary mapping the query index (0-based) to the execute function. + """ + strategy_funcs = {} + + if not os.path.exists(strategy_dir): + print(f"Strategy directory not found: {strategy_dir}") + return {} + + # Get all python files starting with "query_" + files = sorted([f for f in os.listdir(strategy_dir) if f.startswith("query_") and f.endswith(".py")]) + + for filename in files: + # Extract index from filename (e.g., "query_1.py" -> index 0) + try: + # Split "query_1.py" -> ["query", "1.py"] -> "1" -> int(1) -> 0 + idx = int(filename.split('_')[1].split('.')[0]) - 1 + except ValueError: + continue + + file_path = os.path.join(strategy_dir, filename) + module_name = f"strategy_{filename[:-3]}" + + try: + spec = importlib.util.spec_from_file_location(module_name, file_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + if hasattr(module, "execute_query"): + strategy_funcs[idx] = module.execute_query + else: + print(f"WARNING: {filename} has no execute_query(retrieve)") + + except SyntaxError as e: + print(f"ERROR: Syntax error in {filename} (line {e.lineno}): {e.msg}") + print(f" Please fix this file before running execution.") + except Exception as e: + print(f"ERROR: Failed to load {filename}: {e}") + + return strategy_funcs + +def execute_all_strategies(strategies_dir: str, queries_path: str, output_path: str, config: dict): + """ + Main execution loop: + 1. Initializes collection using standard utils. + 2. Loads all strategy scripts. + 3. Runs them against the queries using a wrapped retrieve function. + 4. Saves results. + """ + # 1. Initialize Collection (using chroma_utils) + print("Initializing ChromaDB collection...") + collection = get_db_collection(config) + + # 2. Load gold queries + try: + with open(queries_path, "r", encoding="utf-8") as f: + queries = [json.loads(line) for line in f if line.strip()] + except FileNotFoundError: + print(f"Error: Queries file not found at {queries_path}") + return + + # 3. Load strategies + strategy_funcs = load_strategy_functions(strategies_dir) + print(f"Found {len(strategy_funcs)} valid strategy functions.") + + os.makedirs(os.path.dirname(output_path), exist_ok=True) + + def retrieve_wrapper(query_text, k): + return retrieve(collection, query_text, k, include_chunks=False) + + # 4. Execute + processed_count = 0 + with open(output_path, "w", encoding="utf-8") as f_out: + for i, q_data in enumerate(tqdm(queries, desc="Executing Strategies")): + query_text = q_data["query"] + if i in strategy_funcs: + try: + final_docs_set = strategy_funcs[i](retrieve_wrapper) + final_docs = list(final_docs_set) if isinstance(final_docs_set, set) else final_docs_set + prediction = { + "query": query_text, + "docs": final_docs + } + f_out.write(json.dumps(prediction) + "\n") + processed_count += 1 + except Exception as e: + print(f"Runtime Error in strategy {i+1}: {e}") + else: + pass + + print(f"Execution complete. {processed_count}/{len(queries)} queries processed.") + print(f"Results saved to {output_path}") \ No newline at end of file diff --git a/baseline_quest/lib/decomposition_generator.py b/baseline_quest/lib/decomposition_generator.py new file mode 100644 index 0000000..4d4e433 --- /dev/null +++ b/baseline_quest/lib/decomposition_generator.py @@ -0,0 +1,153 @@ +import os +import json +import dspy +from typing import List, Dict, Any, Optional +from dataclasses import dataclass + +# --- Data Structures --- + +@dataclass +class QuestQuery: + query: str + docs: List[str] + original_query: str + metadata: Optional[Dict[str, Any]] = None + +# --- DSPy Module for Query Decomposition --- + +class QueryDecomposerWithPython(dspy.Signature): + query = dspy.InputField(desc="A natural language retrieval task with implicit set operations.") + target_k = dspy.InputField(desc="The target number of final documents to return. The final list will NOT be truncated to this k.") + + output = dspy.OutputField( + desc=( + "A Python program that decomposes the query into a sequence of smaller retrieval " + "tasks and combines the results using set operations (&, |, -). The program must " + "call the function retrieve(query_string, k), where k is the top-k documents to " + "retrieve and the function returns a set of document IDs (str). " + "Tune k according to the mix of set operations, using larger k when more restrictive " + "operations (&, -) are involved. " + "Return the results from the set operations as a list, without truncation by target_k. " + "Store the final list of document IDs in a variable named result. " + "The last line of the program MUST be: return result. " + "Write ONLY the Python code inside the function body. Do NOT write `def execute_query(...)`. " + "Wrap the code block in and ." + ) + ) + +class QuestQueryDecomposer(dspy.Module): + few_shot_examples = [ + dspy.Example( + query="Birds of Kolombangara or of the Western Province (Solomon Islands)", + target_k=50, + output=""" +birds_of_kolombangara_ids = retrieve('Birds of Kolombangara', k=100) +western_province_birds_ids = retrieve('Birds of the Western Province (Solomon Islands)', k=100) +final_doc_ids = birds_of_kolombangara_ids | western_province_birds_ids +result = list(final_doc_ids) +return result +""" + ), + dspy.Example( + query="Trees of South Africa that are also in the south-central Pacific", + target_k=50, + output=""" +trees_sa_ids = retrieve('Trees of South Africa', k=100) +trees_scp_ids = retrieve('south-central Pacific', k=100) +final_doc_ids = trees_sa_ids & trees_scp_ids +result = list(final_doc_ids) +return result +""" + ), + dspy.Example( + query="2010's adventure films set in the Southwestern United States but not in California", + target_k=50, + output=""" +adventure_films_ids = retrieve('2010s adventure films', k=200) +sw_us_films_ids = retrieve('films set in the Southwestern United States', k=200) +california_films_ids = retrieve('films set in California', k=200) +inclusive_films = adventure_films_ids & sw_us_films_ids +final_doc_ids = inclusive_films - california_films_ids +result = list(final_doc_ids) +return result +""" + ) + ] + + def __init__(self): + super().__init__() + self.generate_code = dspy.Predict(QueryDecomposerWithPython, n=1, temperature=1.0) + + def forward(self, query: str, target_k: int): + return self.generate_code(query=query, target_k=target_k, Demos=self.few_shot_examples) + +def generate_strategies(queries_path: str, output_dir: str, target_k: int, llm_model: str): + """ + Generates Python strategy files for each query in the input file using DSPy. + """ + api_key = os.getenv("OPENAI_API_KEY") + if not api_key: + print("FATAL ERROR: OPENAI_API_KEY environment variable not set.") + return + + # 1. Configure DSPy + try: + lm = dspy.LM( + f'openai/{llm_model}', + temperature=1.0, + max_tokens=2000, + api_key=api_key + ) + dspy.configure(lm=lm) + print(f"DSPy configured with model: {llm_model}") + except Exception as e: + print(f"Error configuring DSPy: {e}") + return + + # 2. Load Queries + queries = [] + try: + with open(queries_path, "r", encoding="utf-8") as f: + for line in f: + if line.strip(): + data = json.loads(line) + # Safe load into dataclass + queries.append(QuestQuery(**{k: v for k, v in data.items() if k in QuestQuery.__annotations__})) + except FileNotFoundError: + print(f"Error: Query file not found at {queries_path}") + return + + # 3. Prepare Output Directory + os.makedirs(output_dir, exist_ok=True) + decomposer = QuestQueryDecomposer() + + print(f"Generating {len(queries)} strategies into '{output_dir}'...") + + # 4. Generate Strategies Loop + generated_count = 0 + for i, quest_query in enumerate(queries): + try: + # Generate code using DSPy + result = decomposer(query=quest_query.query, target_k=target_k) + + # Clean up the output to extract pure Python code + python_code = result.output.strip().replace("", "").replace("", "").strip() + + # Write to file + filename = f"query_{i+1}.py" + filepath = os.path.join(output_dir, filename) + + with open(filepath, "w", encoding="utf-8") as f_out: + f_out.write(f"# Strategy for Query {i+1}: {quest_query.query}\n") + # We wrap the code in a function definition + f_out.write(f'def execute_query(retrieve):\n') + # Indent the generated code + f_out.write(" " + python_code.replace("\n", "\n ") + "\n") + + generated_count += 1 + + except Exception as e: + print(f" [Error] Query {i+1} ('{quest_query.query}'): {e}") + continue + + print(f"Successfully generated {generated_count}/{len(queries)} strategy files.") \ No newline at end of file diff --git a/baseline_quest/lib/retrieval_analyzer.py b/baseline_quest/lib/retrieval_analyzer.py new file mode 100644 index 0000000..4a4f709 --- /dev/null +++ b/baseline_quest/lib/retrieval_analyzer.py @@ -0,0 +1,117 @@ +import os +import numpy as np +import logging +from typing import List, Dict, Union, Any + +from chroma_utils import read_jsonl + +def calculate_recall_stats(gold_path: str, pred_path: str) -> Dict[str, Any]: + """ + Calculates Recall metrics for predictions against a gold standard. + """ + if not os.path.exists(gold_path): + raise FileNotFoundError(f"Gold file not found: {gold_path}") + if not os.path.exists(pred_path): + raise FileNotFoundError(f"Prediction file not found: {pred_path}") + + # Load Data + gold_data = list(read_jsonl(gold_path)) + pred_data = list(read_jsonl(pred_path)) + + # Map predictions by query for O(1) lookup + pred_map = {item['query']: item for item in pred_data} + + stats = { + "per_query": [], + "summary": {}, + "missing_count": 0 + } + + recall_scores = [] + pred_sizes = [] + + for gold in gold_data: + query = gold['query'] + gold_docs = set(gold.get('docs', [])) + + if not gold_docs: + stats["per_query"].append({"query": query, "error": "Gold example has no docs"}) + continue + + if query not in pred_map: + stats["missing_count"] += 1 + recall_scores.append(0.0) + pred_sizes.append(0) + stats["per_query"].append({"query": query, "error": "Missing in predictions"}) + continue + + raw_pred = pred_map[query].get('docs', []) + pred_docs = [] + for d in raw_pred: + if isinstance(d, dict): + pred_docs.append(d.get('title', '')) + else: + pred_docs.append(d) + + pred_set = set(pred_docs) + intersection = gold_docs.intersection(pred_set) + + recall = len(intersection) / len(gold_docs) if len(gold_docs) > 0 else 0 + + query_res = { + "query": query, + "gold_count": len(gold_docs), + "recall": recall, + "pred_size": len(pred_docs), + "covered": len(intersection) + } + + recall_scores.append(recall) + pred_sizes.append(len(pred_docs)) + stats["per_query"].append(query_res) + + if recall_scores: + stats["summary"]["avg_recall"] = np.mean(recall_scores) + stats["summary"]["avg_pred_size"] = np.mean(pred_sizes) + else: + stats["summary"]["avg_recall"] = 0.0 + stats["summary"]["avg_pred_size"] = 0.0 + + return stats + +def write_analysis_report(stats: Dict, output_path: str): + """Writes the calculated stats to a human-readable text file.""" + os.makedirs(os.path.dirname(output_path), exist_ok=True) + + with open(output_path, "w", encoding="utf-8") as f: + # 1. Summary Section + f.write("--- Analysis Report ---\n\n") + f.write("Summary Metrics:\n") + for key, val in stats['summary'].items(): + f.write(f" {key}: {val:.4f}\n") + f.write(f" Missing Queries: {stats['missing_count']}\n\n") + + # 2. Detailed Per-Query Section + f.write("Detailed Results:\n") + for item in stats['per_query']: + q = item.get('query', '') + + if 'error' in item: + f.write(f" [ERROR] {item['error']} | Query: \"{q}\"\n") + else: + recall = item.get('recall', 0.0) + size = item.get('pred_size', 0) + covered = item.get('covered', 0) + gold_count = item.get('gold_count', 0) + + # Format: Recall=0.80 (Size: 50) (4/5) | Query: "..." + f.write(f" - Recall={recall:.2f} (Size: {size}) ({covered}/{gold_count}) | Query: \"{q}\"\n") + + print(f"Report written to: {output_path}") + +def analyze_results(gold_path: str, pred_path: str, output_report_path: str): + """ + Main entry point used by the pipeline script. + """ + stats = calculate_recall_stats(gold_path, pred_path) + write_analysis_report(stats, output_report_path) \ No newline at end of file diff --git a/baseline_quest/lib/retriever.py b/baseline_quest/lib/retriever.py new file mode 100644 index 0000000..504e4d6 --- /dev/null +++ b/baseline_quest/lib/retriever.py @@ -0,0 +1,85 @@ +import logging +import sys +from typing import Dict, List, Set, Union + +__import__('pysqlite3') +sys.modules['sqlite3'] = sys.modules.pop('pysqlite3') + +logger = logging.getLogger(__name__) + +# retrieve single query (used for decompose + retrieve) +def retrieve(collection, query: str, k: int, include_chunks: bool = False) -> Union[Set[str], Dict[str, str]]: + """ + Retrieves k results for a single query string. + """ + if k <= 0: + return {} if include_chunks else set() + + include_fields = ["metadatas"] + if include_chunks: + include_fields.append("documents") + + results = collection.query( + query_texts=[query], + n_results=k, + include=include_fields + ) + + return _process_single_result(results, 0, include_chunks) + + +# batch retrieval (used for retrieve) +def retrieve_batch( + collection, + queries: List[str], + k: int, + include_chunks: bool = False +) -> List[Union[Set[str], Dict[str, str]]]: + """ + Retrieves k results for a list of queries efficiently (vectorized). + """ + if not queries or k <= 0: + return [] + + include_fields = ["metadatas"] + if include_chunks: + include_fields.append("documents") + + results = collection.query( + query_texts=queries, + n_results=k, + include=include_fields + ) + + batch_output = [] + for i in range(len(queries)): + processed = _process_single_result(results, i, include_chunks) + batch_output.append(processed) + + return batch_output + + +# helper function to deduplicate the results +def _process_single_result(results, index, include_chunks): + """ + Internal helper to extract and deduplicate logic for a specific query index. + """ + if not results.get("metadatas") or not results["metadatas"][index]: + return {} if include_chunks else set() + + metas = results["metadatas"][index] + docs_text = results["documents"][index] if include_chunks else [] + + if include_chunks: + title_to_chunk = {} + for i, meta in enumerate(metas): + title = meta.get("title", "No Title") + if title not in title_to_chunk: + title_to_chunk[title] = docs_text[i] + return title_to_chunk + else: + unique_titles = set() + for meta in metas: + if "title" in meta: + unique_titles.add(meta["title"]) + return unique_titles \ No newline at end of file diff --git a/baseline_quest/semantic_retrieval.py b/baseline_quest/semantic_retrieval.py new file mode 100644 index 0000000..f2713fa --- /dev/null +++ b/baseline_quest/semantic_retrieval.py @@ -0,0 +1,88 @@ +import sys +import os +import json +import logging +import yaml +import argparse + +try: + __import__('pysqlite3') + sys.modules['sqlite3'] = sys.modules.pop('pysqlite3') +except ImportError: + pass + +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +# Import from lib +from lib.chroma_utils import get_db_collection, read_jsonl +from lib.retriever import retrieve_batch +from lib.retrieval_analyzer import analyze_results + +CONFIG_PATH = "config.yaml" + +def load_config(path: str): + if not os.path.exists(path): + raise FileNotFoundError(f"Config file not found: {path}") + with open(path, 'r') as f: + return yaml.safe_load(f) + +def main(): + # 1. Load Configuration + logger.info(f"Loading configuration from {CONFIG_PATH}...") + config = load_config(CONFIG_PATH) + + queries_path = config['data']['queries_file'] + top_k = config['retrieval'].get('top_k', 100) + include_chunks = config['retrieval'].get('include_chunks', False) + is_limited = config['indexing'].get('index_first_512', False) + + suffix = "_limited" if is_limited else "" + subset_name = os.path.splitext(os.path.basename(queries_path))[0] + output_pred_path = f"results/semantic_retrieval/pred_unranked{suffix}_{subset_name}.jsonl" + output_report_path = f"results/semantic_retrieval/recall_report{suffix}_{subset_name}.txt" + + # 2. Initialize Collection + logger.info("Initializing ChromaDB collection...") + collection = get_db_collection(config) + + # 3. Load Queries + logger.info(f"Reading queries from {queries_path}...") + queries_data = list(read_jsonl(queries_path)) + + query_texts = [item['query'] for item in queries_data] + logger.info(f"Found {len(query_texts)} queries. Starting retrieval (k={top_k}, chunks={include_chunks})...") + + # 4. Run Batch Retrieval + results = retrieve_batch( + collection, + query_texts, + k=top_k, + include_chunks=include_chunks + ) + + # 5. Write Predictions + logger.info(f"Writing predictions to {output_pred_path}...") + os.makedirs(os.path.dirname(output_pred_path), exist_ok=True) + + with open(output_pred_path, 'w', encoding='utf-8') as f_out: + for q_text, res in zip(query_texts, results): + if include_chunks: + # result: Dict[title, chunk] -> Convert to [{"title": t, "chunk": c}, ...] + docs_output = [{"title": t, "chunk": c} for t, c in res.items()] + else: + # result: Set[title] -> Convert to ["title1", "title2", ...] + docs_output = list(res) + + f_out.write(json.dumps({ + "query": q_text, + "docs": docs_output + }) + "\n") + + # 6. Analyze Results + logger.info("Calculating recall statistics...") + analyze_results(queries_path, output_pred_path, output_report_path) + logger.info(f'Semantic retrieval completed successfully. Results saved to{output_report_path}') + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/baseline_quest/vector_similarity.py b/baseline_quest/vector_similarity.py new file mode 100644 index 0000000..b0ecc60 --- /dev/null +++ b/baseline_quest/vector_similarity.py @@ -0,0 +1,328 @@ +import argparse +import logging +import os +import sys +import torch +import yaml +from typing import Dict, List +from sentence_transformers import SentenceTransformer, util + +from lib.chroma_utils import read_jsonl + +try: + __import__('pysqlite3') + sys.modules['sqlite3'] = sys.modules.pop('pysqlite3') +except ImportError: + pass + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + datefmt='%H:%M:%S' +) + +logger = logging.getLogger(__name__) + +def get_gold_chunk(gold_item: Dict, title: str, query: str) -> str: + """Extracts the 'best' gold chunk from the complex attributions field.""" + try: + attributions = gold_item.get('metadata', {}).get('attributions', {}) + attrib_list = attributions.get(title, []) + + if not attrib_list: + return None + + # The attribution is a list, usually of one dict + attrib_dict = attrib_list[0] + + # Ideal case: the query itself is the key + if query in attrib_dict: + return attrib_dict[query] + + # Fallback: concatenate all chunk snippets + return " ".join(attrib_dict.values()) + + except Exception as e: + logger.warning(f"Error parsing gold chunk for {title}: {e}") + return None + +def log_and_write(f_handle, message: str): + """Helper to write to the report file and log to console.""" + logger.info(message) + f_handle.write(message + "\n") + +def run_similarity_analysis( + gold_path: str, + retrieved_path: str, + output_path: str, + target_query: str, + unrelated_query: str, + model_name: str +): + """Main execution logic for similarity analysis.""" + device = "cuda" if torch.cuda.is_available() else "cpu" + + # 1. Validation + if not os.path.isfile(gold_path): + logger.error(f"Gold file not found at '{gold_path}'") + return + if not os.path.isfile(retrieved_path): + logger.error(f"Retrieved file not found at '{retrieved_path}'") + return + + # 2. Load Model + logger.info(f"Loading embedding model '{model_name}' on device '{device}'...") + try: + model = SentenceTransformer(model_name, device=device) + except Exception as e: + logger.error(f"Failed to load model: {e}") + return + + # 3. Load Data using chroma_utils.read_jsonl + logger.info("Loading dataset files...") + try: + gold_data = {item['query']: item for item in read_jsonl(gold_path)} + retrieved_data = {item['query']: item for item in read_jsonl(retrieved_path)} + except Exception as e: + logger.error(f"Error reading JSONL files: {e}") + return + + # 4. Check Query Existence + if target_query not in gold_data: + logger.error(f"Target query not found in GOLD file: '{target_query}'") + return + if target_query not in retrieved_data: + logger.error(f"Target query not found in RETRIEVED file: '{target_query}'") + return + + gold_item = gold_data[target_query] + retrieved_item = retrieved_data[target_query] + + gold_titles = set(gold_item.get("docs", [])) + retrieved_docs_list = retrieved_item.get("docs", []) + + # Map retrieved chunks for lookup + retrieved_chunks_map = { + d['title']: d['chunk'] + for d in retrieved_docs_list if d.get('title') + } + + # Identify False Positives (documents retrieved but not in gold set) + fp_data = [ + {'title': d['title'], 'chunk': d['chunk']} + for d in retrieved_docs_list + if d.get('title') not in gold_titles and d.get('chunk') + ] + rep_fp_doc = fp_data[0] if fp_data else None + + # Load Unrelated Data (Baseline) + unrelated_docs_data = [] + if unrelated_query in retrieved_data: + unrelated_item = retrieved_data[unrelated_query] + unrelated_list = unrelated_item.get('docs', []) + if unrelated_list: + for doc in unrelated_list: + unrelated_docs_data.append({ + 'query': unrelated_query, + 'title': doc.get('title'), + 'chunk': doc.get('chunk') + }) + else: + logger.warning(f"Unrelated query found, but has no docs: '{unrelated_query}'") + else: + logger.warning(f"Unrelated query not found in retrieved file: '{unrelated_query}'") + + # 5. Prepare Batch Encoding + texts_to_encode = [target_query] + text_info = [] + + # A. Gold Documents + for title in gold_titles: + gold_chunk = get_gold_chunk(gold_item, title, target_query) + if not gold_chunk: + logger.warning(f"Skipping {title}: No gold chunk text found.") + continue + + texts_to_encode.append(gold_chunk) + text_info.append({'title': title, 'type': 'gold_chunk', 'chunk': gold_chunk}) + + # Check if retrieved (True Positive) or missed (False Negative) + if title in retrieved_chunks_map: + retrieved_chunk = retrieved_chunks_map[title] + texts_to_encode.append(retrieved_chunk) + text_info.append({'title': title, 'type': 'retrieved_chunk', 'chunk': retrieved_chunk}) + else: + # If missed, compare against a "Representative False Positive" if available + if rep_fp_doc: + texts_to_encode.append(rep_fp_doc['chunk']) + text_info.append({ + 'title': title, # Linked to gold title context + 'type': 'rep_fp_chunk', + 'chunk': rep_fp_doc['chunk'], + 'fp_title': rep_fp_doc['title'] + }) + + # B. Pure False Positives (not linked to any gold doc) + other_fp_data = fp_data[1:] if rep_fp_doc else fp_data + for fp_doc in other_fp_data: + texts_to_encode.append(fp_doc['chunk']) + text_info.append({'title': fp_doc['title'], 'type': 'pure_fp_chunk', 'chunk': fp_doc['chunk']}) + + # C. Unrelated Chunks + for doc in unrelated_docs_data: + if doc['chunk']: + texts_to_encode.append(doc['chunk']) + text_info.append({ + 'title': doc['title'], + 'type': 'unrelated_chunk', + 'chunk': doc['chunk'], + 'orig_query': doc['query'] + }) + + # 6. Calculate Similarities + logger.info(f"Encoding {len(texts_to_encode)} texts...") + embeddings = model.encode(texts_to_encode, convert_to_tensor=True, normalize_embeddings=True) + + query_embedding = embeddings[0] + chunk_embeddings = embeddings[1:] + + # Compute cosine similarity + similarities = util.cos_sim(query_embedding, chunk_embeddings)[0].tolist() + + # 7. Generate Report + os.makedirs(os.path.dirname(output_path), exist_ok=True) + + with open(output_path, "w", encoding="utf-8") as f: + log_and_write(f, "--- Similarity Analysis Report ---") + log_and_write(f, f"Query: {target_query}") + log_and_write(f, f"Gold Titles: {list(gold_titles)}\n") + log_and_write(f, "="*80) + + # Organize results + report_map = {title: {} for title in gold_titles} + pure_fps = [] + unrelateds = [] + + sim_idx = 0 + for info in text_info: + sim = similarities[sim_idx] + sim_idx += 1 + + t_type = info['type'] + title = info['title'] + + if t_type == 'gold_chunk': + report_map[title]['gold'] = {'sim': sim, 'chunk': info['chunk']} + elif t_type == 'retrieved_chunk': + report_map[title]['retrieved'] = {'sim': sim, 'chunk': info['chunk']} + elif t_type == 'rep_fp_chunk': + report_map[title]['fp'] = {'sim': sim, 'chunk': info['chunk'], 'title': info['fp_title']} + elif t_type == 'pure_fp_chunk': + pure_fps.append({'sim': sim, 'chunk': info['chunk'], 'title': title}) + elif t_type == 'unrelated_chunk': + unrelateds.append({'sim': sim, 'chunk': info['chunk'], 'title': title}) + + # Write Analysis: Gold Docs + log_and_write(f, "--- Analysis of Gold Set Documents ---") + for title in gold_titles: + log_and_write(f, f"\nDocument: {title}") + data = report_map.get(title, {}) + + if 'gold' in data: + g_sim = data['gold']['sim'] + g_chunk = data['gold']['chunk'][:200].replace('\n', ' ') + log_and_write(f, f" Sim (Query <-> Gold Chunk): {g_sim:.4f}") + log_and_write(f, f" Chunk: {g_chunk}...") + else: + log_and_write(f, " [!] Error: Gold chunk missing.") + continue + + if 'retrieved' in data: + r_sim = data['retrieved']['sim'] + r_chunk = data['retrieved']['chunk'][:200].replace('\n', ' ') + log_and_write(f, f" Sim (Query <-> Retrieved Chunk): {r_sim:.4f}") + log_and_write(f, f" Chunk: {r_chunk}...") + log_and_write(f, f" Status: TRUE POSITIVE") + elif 'fp' in data: + fp_sim = data['fp']['sim'] + fp_chunk = data['fp']['chunk'][:200].replace('\n', ' ') + fp_title = data['fp']['title'] + log_and_write(f, f" Sim (Query <-> Random FP Chunk): {fp_sim:.4f}") + log_and_write(f, f" FP Title: {fp_title}") + log_and_write(f, f" Chunk: {fp_chunk}...") + log_and_write(f, f" Status: FALSE NEGATIVE (Doc not retrieved)") + else: + log_and_write(f, f" Status: FALSE NEGATIVE (No FP available for comparison)") + + # Write Analysis: Other FPs + log_and_write(f, "\n" + "="*80) + log_and_write(f, "--- Analysis of Other False Positives (Not in Gold Set) ---") + if pure_fps: + pure_fps.sort(key=lambda x: x['sim'], reverse=True) + for item in pure_fps: + log_and_write(f, f"\n Sim: {item['sim']:.4f} | Title: {item['title']}") + log_and_write(f, f" Chunk: {item['chunk'][:200].replace('\n', ' ')}...") + else: + log_and_write(f, " None found.") + + # Write Analysis: Unrelated + log_and_write(f, "\n" + "="*80) + log_and_write(f, "--- Analysis of Unrelated Chunks (Baseline) ---") + log_and_write(f, f" (Chunks from query: '{unrelated_query}')") + + if unrelateds: + unrelateds.sort(key=lambda x: x['sim'], reverse=True) + for item in unrelateds: + log_and_write(f, f"\n Sim: {item['sim']:.4f} | Title: {item['title']}") + log_and_write(f, f" Chunk: {item['chunk'][:200].replace('\n', ' ')}...") + else: + log_and_write(f, " No unrelated chunks found.") + + logger.info(f"Report saved to: {output_path}") + +def main(): + # Load Config to set dynamic defaults + config_path = "config.yaml" + if not os.path.exists(config_path): + logger.error("config.yaml not found.") + sys.exit(1) + + with open(config_path, "r") as f: + config = yaml.safe_load(f) + + # Config values for defaults + default_gold_path = config['data']['queries_file'] # Uses the queries_file from config.yaml + default_model_name = config['indexing']['embedding_model'] + + parser = argparse.ArgumentParser(description="Analyze embedding similarity between Query, Gold Chunks, and Retrieved Chunks.") + + # Defaults are updated to use config values + parser.add_argument("--gold", type=str, default=default_gold_path, + help=f"Path to gold standard JSONL file (default: {default_gold_path}).") + parser.add_argument("--retrieved", type=str, default="full_document/pred_unranked.jsonl", + help="Path to retrieved predictions JSONL file.") + parser.add_argument("--out", type=str, default="gold_vs_retrieved_similarity_report.txt", + help="Path to output text report.") + parser.add_argument("--query", type=str, + default="cultural geography and Science books but not about creativity", + help="The specific query string to analyze.") + parser.add_argument("--unrelated-query", type=str, + default="1947 Science Linguistics books", + help="An unrelated query string to fetch baseline chunks.") + parser.add_argument("--model", type=str, default=default_model_name, + help=f"HuggingFace model name for embeddings (default: {default_model_name}).") + + args = parser.parse_args() + + run_similarity_analysis( + gold_path=args.gold, + retrieved_path=args.retrieved, + output_path=args.out, + target_query=args.query, + unrelated_query=args.unrelated_query, + model_name=args.model + ) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 8e7bc1a..c22a18b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,11 +6,13 @@ readme = "README.md" requires-python = ">=3.10" keywords = ["deep research", "optimization", "llm", "AI programming", "query engine"] authors = [ - {name="MIT DSG", email="mdrusso@csail.mit.edu"}, + { name = "MIT DSG", email = "mdrusso@csail.mit.edu" }, ] dependencies = [ "beautifulsoup4>=4.13.4", "colorama>=0.4.6", + "dspy>=2.5.0", + "fastapi~=0.115.0", "chromadb>=1.3.4", "fastapi>=0.115.0", "google-cloud-aiplatform>=1.38", @@ -27,13 +29,16 @@ dependencies = [ "setuptools>=70.1.1", "smolagents[toolkit]", "tiktoken>=0.8.0", + "numpy>=1.24.0", + "scikit-learn>=1.3.0", + "sentence-transformers>=2.2.0", ] -classifiers=[ - "Development Status :: 4 - Beta", # Change as appropriate +classifiers = [ + "Development Status :: 4 - Beta", # Change as appropriate "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", # Change as appropriate + "License :: OSI Approved :: MIT License", # Change as appropriate "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", # Specify versions you support + "Programming Language :: Python :: 3.8", # Specify versions you support # Add more classifiers as appropriate ] @@ -43,7 +48,7 @@ vllm = [ ] [tool.setuptools] -package-dir = {"" = "src"} +package-dir = { "" = "src" } include-package-data = true [tool.setuptools.packages.find] diff --git a/quest_test.py b/quest_test.py new file mode 100644 index 0000000..e4391e9 --- /dev/null +++ b/quest_test.py @@ -0,0 +1,297 @@ +""" +QUEST Dataset Loader for DSPy Testing + +This module provides functionality to load and work with Google's QUEST dataset, +which contains entity-seeking queries with implicit set operations. + +Dataset info: https://github.com/google-research/language/tree/master/language/quest +Paper: https://aclanthology.org/2023.acl-long.784.pdf +""" + +import json +import random +import requests +import re +from typing import List, Dict, Any, Optional +from dataclasses import dataclass +import dspy +from sklearn.cluster import KMeans +from sentence_transformers import SentenceTransformer + + +@dataclass +class QuestQuery: + """Represents a single QUEST query with its metadata.""" + query: str + docs: List[str] # Relevant documents/entities + original_query: str # Query with markup showing set operations + scores: Optional[List[float]] = None + metadata: Optional[Dict[str, Any]] = None + + +class QuestDatasetLoader: + """ + Loads and manages the QUEST dataset for DSPy experiments. + + The QUEST dataset contains natural language queries that implicitly specify + set operations (intersection, union, difference) over entities. + """ + TRAINING_SET_URL = "https://storage.googleapis.com/gresearch/quest/train.jsonl" + + def load_training_data(self) -> List[QuestQuery]: + """ + Fetch content bytes from training set URL and parse into QuestQuery list. + """ + response = requests.get(self.TRAINING_SET_URL) + content_bytes = response.content + + queries = [] + for line_num, line in enumerate(content_bytes.decode('utf-8').strip().split('\n'), 1): + if line.strip(): + try: + data = json.loads(line) + + quest_query = QuestQuery( + query=data.get('query', ''), + docs=data.get('docs', []), + original_query=data.get('original_query', ''), + scores=data.get('scores'), + metadata=data.get('metadata') + ) + + queries.append(quest_query) + except json.JSONDecodeError as e: + print(f"JSON parsing error on line {line_num}: {e}") + print(f"Line content (first 100 chars): {line[:100]}") + continue + return queries + + +class QueryDecomposerWithPython(dspy.Signature): + query = dspy.InputField(desc="Natural language retrieval tasks with implicit set operations") + output = dspy.OutputField( + desc="A Python program that decomposes the query into a sequence of smaller retrieval tasks and combines the results using set operations. The program should call the " + "function retrieve(query_string, k) where k is the top-k results to retrieve, and rerank(documents), which takes a a collection of document IDs and reorders them according to relevance." + "Do not write extra helper functions besides retrieve(), and rerank()." + "Only write the code segment that fill in the following template returns the final result, and wrap the segment in and :" + "def executeQuery(retrieve, rerank):" + " your code " + " return result") + + +class QuestQueryDecomposerWithPython(dspy.Module): + few_shot_examples = [ + dspy.Example( + query="Birds of Kolombangara or of the Western Province (Solomon Islands)", + output="birds_of_kolombangara_ids = retrieve('Birds of Kolombangara', k=100)" + "western_province_birds_ids = retrieve('Birds of the Western Province (Solomon Islands)', k=100)" + "result = rerank(set(birds_of_kolombangara_ids) | set(western_province_birds_ids)) " + ), + dspy.Example( + query="Trees of South Africa that are also in the south-central Pacific", + output=" trees_sa_ids = retrieve('Trees of South Africa', k=200)" + "trees_scp_ids = retrieve('south-central Pacific', k=200)" + "result = rerank(set(trees_sa_ids) & set(trees_scp_ids)) " + ), + dspy.Example( + query="2010's adventure films set in the Southwestern United States but not in California", + output="adventure_films_ids = retrieve('2010s adventure films', k=100)" + "sw_us_films_ids = retrieve('films set in the Southwestern United States', k=100)" + "california_films_ids = retrieve('films set in California', k=1000)" + "inclusive_films = set(adventure_films_ids) & set(sw_us_films_ids)" + "result = rerank(inclusive_films - set(california_films_ids)) " + ) + ] + + def __init__(self): + super().__init__() + self.decompose = dspy.Predict(QueryDecomposerWithPython) + + def forward(self, query: str): + # Use few-shot examples to guide the prediction + return self.decompose(query=query) + + +class QueryDecomposerWithLogic(dspy.Signature): + query = dspy.InputField(desc="Natural language query with implicit set operations") + output = dspy.OutputField(desc="Query with semantic concepts marked using tags, connected with set operations. Only use union (OR), intersection (OR), and set difference (/) in your output. Each should contain a complete searchable concept, not individual words.") + + +class QuestQueryDecomposerWithLogic(dspy.Module): + few_shot_examples = [ + dspy.Example( + query="Birds of Kolombangara or of the Western Province (Solomon Islands)", + output="Birds of Kolombangara OR Birds of the Western Province (Solomon Islands)" + ), + dspy.Example( + query="Trees of South Africa that are also in the south-central Pacific", + output="Flora of the south-central Pacific AND Trees of South Africa" + ), + dspy.Example( + query="2010's adventure films set in the Southwestern United States but not in California", + output="(2010s adventure films AND Films set in the Southwestern United States) / Films set in California" + ) + ] + + def __init__(self): + super().__init__() + self.decompose = dspy.Predict(QueryDecomposerWithLogic) + + + def forward(self, query: str): + # Use few-shot examples to guide the prediction + return self.decompose(query=query) + + +class ConceptExtractor(dspy.Signature): + """Extract a reusable metadata concept from a cluster of similar phrases.""" + cluster_samples = dspy.InputField( + desc="A list of similar phrases from a semantic cluster. These phrases share common characteristics." + ) + concept_name = dspy.OutputField( + desc="A concise, snake_case metadata field name that captures the shared concept (e.g., 'geographic_region', 'film_genre', 'time_period', 'taxonomic_group'). Should be reusable across documents." + ) + concept_description = dspy.OutputField( + desc="A clear 1-2 sentence description of what this metadata field represents and how it should be used for filtering documents." + ) + example_values = dspy.OutputField( + desc="3-5 example values that this metadata field might contain, based on the cluster samples." + ) + + +class ClusterConceptExtractor(dspy.Module): + """DSPy module that analyzes phrase clusters and extracts reusable metadata concepts.""" + few_shot_examples = [ + dspy.Example( + cluster_samples=["Films set in California", "Films set in New York", "Films set in Texas", + "Films in London", "Films set in Paris", "Films set in Tokyo"], + concept_name="filming_location", + concept_description="The geographic location where a film is set or takes place. This field helps users filter content by specific cities, states, or countries.", + example_values=["California", "New York", "London", "Paris", "Tokyo"] + ), + dspy.Example( + cluster_samples=["1990s films", "2000s comedy films", "1980s action films", + "2010s adventure films", "1970s drama films"], + concept_name="release_decade", + concept_description="The decade when the media was released or published. Useful for filtering content by time period or era.", + example_values=["1970s", "1980s", "1990s", "2000s", "2010s"] + ), + dspy.Example( + cluster_samples=["Birds of North America", "Mammals of Europe", "Reptiles of Asia", + "Flora of South America", "Trees of Africa"], + concept_name="taxonomic_geographic_distribution", + concept_description="Describes the geographic distribution of biological species or taxonomic groups. Enables filtering by both organism type and geographic region.", + example_values=["Birds of North America", "Mammals of Europe", "Flora of South America"] + ) + ] + + def __init__(self): + super().__init__() + self.extract = dspy.ChainOfThought(ConceptExtractor) + + + def forward(self, cluster_samples: List[str]): + """Extract concept metadata from a list of clustered phrases.""" + # Format samples as a readable list + samples_text = "\n".join([f"- {sample}" for sample in cluster_samples[:15]]) # Limit to 15 samples + + # Use few-shot examples to guide extraction + return self.extract(cluster_samples=samples_text) + + +def test_llm_decomposition(): + lm = dspy.LM('openai/gpt-5-mini', temperature=1.0, max_tokens=16000, api_key="") + dspy.configure(lm=lm) + + # Initialize the loader + loader = QuestDatasetLoader() + + # Load training data + training_queries = loader.load_training_data() + print(f"Loaded {len(training_queries)} training queries") + + # Sample 10 random queries for testing + sample_queries = random.sample(training_queries, min(10, len(training_queries))) + print(f"Testing on {len(sample_queries)} randomly sampled queries") + + # Initialize the query decomposer + optimizer = dspy.LabeledFewShot(k=3) + decomposer = optimizer.compile(student=QuestQueryDecomposerWithPython(), trainset=QuestQueryDecomposerWithPython.few_shot_examples) + + for i, quest_query in enumerate(sample_queries): + result = decomposer(query=quest_query.query) + predicted = result.output + + print(f"\n--- Example {i + 1} ---") + print(f"Query: {quest_query.query}") + print(f"Original Query: {quest_query.original_query}") + print(f"Decomposed: {predicted}") + + +def test_concept_clustering(): + # Initialize the loader + loader = QuestDatasetLoader() + + # Load training data + training_queries = loader.load_training_data() + print(f"Loaded {len(training_queries)} training queries") + + # Extract phrases used to generate queries from the original queries + # TODO: Think about whether we should use unique phrases, weight phrases, or keep as-is + marked_phrases = [] + for query in training_queries: + if query.original_query: + # Find all phrases between tags + phrases = re.findall(r'(.*?)', query.original_query) + marked_phrases.extend(phrases) + + # Generate embeddings + print(f"\nGenerating embeddings...") + model = SentenceTransformer('all-MiniLM-L6-v2') # Fast and effective model + embeddings = model.encode(marked_phrases, show_progress_bar=True) + print(f"Generated embeddings with shape: {embeddings.shape}") + + # TODO: Determine optimal number of clusters (e.g., using elbow method) + n_clusters = 50 + print(f"\nUsing {n_clusters} clusters") + + # Perform clustering + print(f"Running K-Means clustering...") + kmeans = KMeans(n_clusters=n_clusters, random_state=42, n_init=10) + cluster_labels = kmeans.fit_predict(embeddings) + + # Group phrases by cluster + clusters = {} + for phrase, label in zip(marked_phrases, cluster_labels): + if label not in clusters: + clusters[label] = [] + clusters[label].append(phrase) + + lm = dspy.LM('openai/gpt-5-mini', temperature=1.0, max_tokens=16000, api_key="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") + dspy.configure(lm=lm) + # Initialize the concept extractor + optimizer = dspy.LabeledFewShot(k=3) + concept_extractor = optimizer.compile(student=ClusterConceptExtractor(), + trainset=ClusterConceptExtractor.few_shot_examples) + + for cluster_id, phrases in clusters.items(): + # Use more samples for better concept extraction (up to 10) + sample_size = min(10, len(phrases)) + sampled_phrases = random.sample(phrases, sample_size) + print(f"Cluster {cluster_id} (sampled {sample_size} of {len(phrases)} phrases):") + for i, phrase in enumerate(sampled_phrases, 1): + print(f" {phrase}") + try: + result = concept_extractor(cluster_samples=sampled_phrases) + print(f"\n 📋 Concept Name: {result.concept_name}") + print(f" 📝 Description: {result.concept_description}") + print(f" 💡 Example Values: {result.example_values}") + except Exception as e: + print(f" ⚠️ Error extracting concept: {e}") + + # TODO(Tianyu): Try to distill concepts recursively here -- many of these can clearly be further combined + + +if __name__ == "__main__": + # test_llm_decomposition() + test_concept_clustering()