data_processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-05T22:57:50.467285400Z",
     "start_time": "2024-12-05T22:55:05.582762600Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-12-05 17:55:07,245 - Found 404 bookshelves.\n",
      "2024-12-05 17:55:07,246 - Processing genre: Science Fiction\n",
      "2024-12-05 17:55:07,397 - Skipping non-English book: Voyage au Centre de la Terre (French)\n",
      "Jules Verne\n",
      "1526 downloads\n",
      "2024-12-05 17:55:07,397 - Skipping non-English book: L'île mystérieuse (French)\n",
      "Jules Verne\n",
      "1423 downloads\n",
      "2024-12-05 17:55:07,398 - Found 23 books in the selected bookshelf.\n",
      "2024-12-05 17:55:07,690 - Downloaded: gutenberg_bookshelf_data\\Science_Fiction\\raw\\The Marching Morons C. M. Kornbluth.txt\n",
      "2024-12-05 17:55:08,063 - Downloaded: gutenberg_bookshelf_data\\Science_Fiction\\raw\\The Stolen Bacillus and Other Incidents H. G. Wells.txt\n",
      "2024-12-05 17:55:08,462 - Downloaded: gutenberg_bookshelf_data\\Science_Fiction\\raw\\The Lost World Arthur Conan Doyle.txt\n",
      "2024-12-05 17:55:08,840 - Downloaded: gutenberg_bookshelf_data\\Science_Fiction\\raw\\A Journey to the Centre of the Earth Jules Verne.txt\n",
      "2024-12-05 17:55:09,124 - Failed to download: https://www.gutenberg.org/ebooks/19513.txt.utf-8\n",
      "2024-12-05 17:55:09,445 - Downloaded: gutenberg_bookshelf_data\\Science_Fiction\\raw\\Anthem Ayn Rand.txt\n",
      "2024-12-05 17:55:10,277 - Downloaded: gutenberg_bookshelf_data\\Science_Fiction\\raw\\Flatland_ A Romance of Many Dimensions Edwin Abbott Abbott.txt\n",
      "2024-12-05 17:55:10,514 - Downloaded: gutenberg_bookshelf_data\\Science_Fiction\\raw\\The Eyes Have It Philip K. Dick.txt\n",
      "2024-12-05 17:55:11,113 - Downloaded: gutenberg_bookshelf_data\\Science_Fiction\\raw\\The Mysterious Island Jules Verne.txt\n",
      "2024-12-05 17:55:11,325 - Failed to download: https://www.gutenberg.org/ebooks/20000.txt.utf-8\n",
      "2024-12-05 17:55:11,801 - Downloaded: gutenberg_bookshelf_data\\Science_Fiction\\raw\\Twenty Thousand Leagues under the Sea Jules Verne.txt\n",
      "2024-12-05 17:55:12,173 - Downloaded: gutenberg_bookshelf_data\\Science_Fiction\\raw\\The Lani People Jesse F. Bone.txt\n",
      "2024-12-05 17:55:12,638 - Downloaded: gutenberg_bookshelf_data\\Science_Fiction\\raw\\Twenty Thousand Leagues Under the Seas_ An Underwater Tour of the World Jules Verne.txt\n",
      "2024-12-05 17:55:13,026 - Downloaded: gutenberg_bookshelf_data\\Science_Fiction\\raw\\A Princess of Mars Edgar Rice Burroughs.txt\n",
      "2024-12-05 17:55:13,498 - Downloaded: gutenberg_bookshelf_data\\Science_Fiction\\raw\\The Country of the Blind, and Other Stories H. G. Wells.txt\n",
      "2024-12-05 17:55:13,764 - Downloaded: gutenberg_bookshelf_data\\Science_Fiction\\raw\\2 B R 0 2 B Kurt Vonnegut.txt\n",
      "2024-12-05 17:55:14,343 - Downloaded: gutenberg_bookshelf_data\\Science_Fiction\\raw\\With the Night Mail_ A Story of 2000 A.D. Rudyard Kipling.txt\n",
      "2024-12-05 17:55:14,767 - Downloaded: gutenberg_bookshelf_data\\Science_Fiction\\raw\\The Big Trip Up Yonder Kurt Vonnegut.txt\n",
      "2024-12-05 17:55:15,107 - Downloaded: gutenberg_bookshelf_data\\Science_Fiction\\raw\\The island of Doctor Moreau H. G. Wells.txt\n",
      "2024-12-05 17:55:15,957 - Downloaded: gutenberg_bookshelf_data\\Science_Fiction\\raw\\The Time Machine H. G. Wells.txt\n",
      "2024-12-05 17:55:15,961 - Metadata saved to gutenberg_bookshelf_data\\Science_Fiction/metadata.csv\n",
      "2024-12-05 17:55:16,011 - Processed: 2 B R 0 2 B Kurt Vonnegut.txt\n",
      "2024-12-05 17:55:16,532 - Processed: A Journey to the Centre of the Earth Jules Verne.txt\n",
      "2024-12-05 17:55:16,849 - Processed: A Princess of Mars Edgar Rice Burroughs.txt\n",
      "2024-12-05 17:55:16,971 - Processed: Anthem Ayn Rand.txt\n",
      "2024-12-05 17:55:17,162 - Processed: Flatland_ A Romance of Many Dimensions Edwin Abbott Abbott.txt\n",
      "2024-12-05 17:55:17,330 - Processed: Second Variety Philip K. Dick.txt\n",
      "2024-12-05 17:55:17,373 - Processed: The Big Trip Up Yonder Kurt Vonnegut.txt\n",
      "2024-12-05 17:55:18,346 - Processed: The Country of the Blind, and Other Stories H. G. Wells.txt\n",
      "2024-12-05 17:55:18,375 - Processed: The Eyes Have It Philip K. Dick.txt\n",
      "2024-12-05 17:55:18,633 - Processed: The island of Doctor Moreau H. G. Wells.txt\n",
      "2024-12-05 17:55:19,107 - Processed: The Lani People Jesse F. Bone.txt\n",
      "2024-12-05 17:55:19,554 - Processed: The Lost World Arthur Conan Doyle.txt\n",
      "2024-12-05 17:55:19,647 - Processed: The Marching Morons C. M. Kornbluth.txt\n",
      "2024-12-05 17:55:20,682 - Processed: The Mysterious Island Jules Verne.txt\n",
      "2024-12-05 17:55:21,087 - Processed: The Stolen Bacillus and Other Incidents H. G. Wells.txt\n",
      "2024-12-05 17:55:21,281 - Processed: The Time Machine H. G. Wells.txt\n",
      "2024-12-05 17:55:21,651 - Processed: The War of the Worlds H. G. Wells.txt\n",
      "2024-12-05 17:55:22,368 - Processed: Twenty Thousand Leagues under the Sea Jules Verne.txt\n",
      "2024-12-05 17:55:23,224 - Processed: Twenty Thousand Leagues Under the Seas_ An Underwater Tour of the World Jules Verne.txt\n",
      "2024-12-05 17:55:23,336 - Processed: With the Night Mail_ A Story of 2000 A.D. Rudyard Kipling.txt\n",
      "2024-12-05 17:55:23,338 - Linguistic features saved to gutenberg_bookshelf_data\\Science_Fiction\\cleaned/linguistic_features.csv\n",
      "2024-12-05 17:55:23,339 - Processing genre: Fantasy\n",
      "2024-12-05 17:55:23,662 - Found 25 books in the selected bookshelf.\n",
      "2024-12-05 17:55:24,055 - Downloaded: gutenberg_bookshelf_data\\Fantasy\\raw\\Le Morte d'Arthur_ Volume 1 Sir Thomas Malory.txt\n",
      "2024-12-05 17:55:24,292 - Failed to download: https://www.gutenberg.org/ebooks/19466.txt.utf-8\n",
      "2024-12-05 17:55:25,560 - Downloaded: gutenberg_bookshelf_data\\Fantasy\\raw\\The Merry Adventures of Robin Hood Howard Pyle.txt\n",
      "2024-12-05 17:55:27,040 - Downloaded: gutenberg_bookshelf_data\\Fantasy\\raw\\The Story of Sigurd the Volsung and the Fall of the Niblungs William Morris.txt\n",
      "2024-12-05 17:55:27,377 - Downloaded: gutenberg_bookshelf_data\\Fantasy\\raw\\The Wonderful Wizard of Oz L. Frank Baum.txt\n",
      "2024-12-05 17:55:27,681 - Downloaded: gutenberg_bookshelf_data\\Fantasy\\raw\\The Book of Wonder Lord Dunsany.txt\n",
      "2024-12-05 17:55:28,239 - Downloaded: gutenberg_bookshelf_data\\Fantasy\\raw\\The Gods of Pegana Lord Dunsany.txt\n",
      "2024-12-05 17:55:29,917 - Downloaded: gutenberg_bookshelf_data\\Fantasy\\raw\\The Well at the World's End_ A Tale William Morris.txt\n",
      "2024-12-05 17:55:30,318 - Downloaded: gutenberg_bookshelf_data\\Fantasy\\raw\\The Story of the Volsungs (Volsunga Saga); with Excerpts from the Poetic Edda.txt\n",
      "2024-12-05 17:55:31,564 - Downloaded: gutenberg_bookshelf_data\\Fantasy\\raw\\The Legends of King Arthur and His Knights Sir Thomas Malory and Sir James Knowles.txt\n",
      "2024-12-05 17:55:31,776 - Failed to download: https://www.gutenberg.org/ebooks/19450.txt.utf-8\n",
      "2024-12-05 17:55:32,210 - Downloaded: gutenberg_bookshelf_data\\Fantasy\\raw\\The Night Land William Hope Hodgson.txt\n",
      "2024-12-05 17:55:32,678 - Downloaded: gutenberg_bookshelf_data\\Fantasy\\raw\\The Merry Adventures of Robin Hood Howard Pyle.txt\n",
      "2024-12-05 17:55:33,196 - Downloaded: gutenberg_bookshelf_data\\Fantasy\\raw\\Four Arthurian Romances active 12th century de Troyes Chrétien.txt\n",
      "2024-12-05 17:55:34,301 - Downloaded: gutenberg_bookshelf_data\\Fantasy\\raw\\Men of Iron Howard Pyle.txt\n",
      "2024-12-05 17:55:36,075 - Downloaded: gutenberg_bookshelf_data\\Fantasy\\raw\\Le Morte d'Arthur_ Volume 2 Sir Thomas Malory.txt\n",
      "2024-12-05 17:55:36,948 - Downloaded: gutenberg_bookshelf_data\\Fantasy\\raw\\The Lost Princess of Oz L. Frank Baum.txt\n",
      "2024-12-05 17:55:37,292 - Downloaded: gutenberg_bookshelf_data\\Fantasy\\raw\\The House on the Borderland William Hope Hodgson.txt\n",
      "2024-12-05 17:55:37,681 - Downloaded: gutenberg_bookshelf_data\\Fantasy\\raw\\The Marvelous Land of Oz L. Frank Baum.txt\n",
      "2024-12-05 17:55:38,075 - Downloaded: gutenberg_bookshelf_data\\Fantasy\\raw\\Irish Fairy Tales James Stephens.txt\n",
      "2024-12-05 17:55:38,078 - Metadata saved to gutenberg_bookshelf_data\\Fantasy/metadata.csv\n",
      "2024-12-05 17:55:38,343 - Processed: Dorothy and the Wizard in Oz L. Frank Baum.txt\n",
      "2024-12-05 17:55:39,385 - Processed: Four Arthurian Romances active 12th century de Troyes Chrétien.txt\n",
      "2024-12-05 17:55:39,766 - Processed: Irish Fairy Tales James Stephens.txt\n",
      "2024-12-05 17:55:40,560 - Processed: Le Morte d'Arthur_ Volume 1 Sir Thomas Malory.txt\n",
      "2024-12-05 17:55:41,469 - Processed: Le Morte d'Arthur_ Volume 2 Sir Thomas Malory.txt\n",
      "2024-12-05 17:55:41,843 - Processed: Men of Iron Howard Pyle.txt\n",
      "2024-12-05 17:55:42,231 - Processed: News from Nowhere; Or, An Epoch of Rest William Morris.txt\n",
      "2024-12-05 17:55:42,351 - Processed: The Book of Wonder Lord Dunsany.txt\n",
      "2024-12-05 17:55:42,437 - Processed: The Gods of Pegana Lord Dunsany.txt\n",
      "2024-12-05 17:55:42,742 - Processed: The House on the Borderland William Hope Hodgson.txt\n",
      "2024-12-05 17:55:43,214 - Processed: The Legends of King Arthur and His Knights Sir Thomas Malory and Sir James Knowles.txt\n",
      "2024-12-05 17:55:43,465 - Processed: The Lost Princess of Oz L. Frank Baum.txt\n",
      "2024-12-05 17:55:43,695 - Processed: The Marvelous Land of Oz L. Frank Baum.txt\n",
      "2024-12-05 17:55:44,280 - Processed: The Merry Adventures of Robin Hood Howard Pyle.txt\n",
      "2024-12-05 17:55:45,081 - Processed: The Night Land William Hope Hodgson.txt\n",
      "2024-12-05 17:55:45,613 - Processed: The Story of Sigurd the Volsung and the Fall of the Niblungs William Morris.txt\n",
      "2024-12-05 17:55:45,894 - Processed: The Story of the Volsungs (Volsunga Saga); with Excerpts from the Poetic Edda.txt\n",
      "2024-12-05 17:55:46,062 - Processed: The Sword of Welleran and Other Stories Lord Dunsany.txt\n",
      "2024-12-05 17:55:47,087 - Processed: The Well at the World's End_ A Tale William Morris.txt\n",
      "2024-12-05 17:55:47,296 - Processed: The Wonderful Wizard of Oz L. Frank Baum.txt\n",
      "2024-12-05 17:55:47,298 - Linguistic features saved to gutenberg_bookshelf_data\\Fantasy\\cleaned/linguistic_features.csv\n",
      "2024-12-05 17:55:47,299 - Processing genre: Mystery Fiction\n",
      "2024-12-05 17:55:47,562 - Found 13 books in the selected bookshelf.\n",
      "2024-12-05 17:55:48,058 - Downloaded: gutenberg_bookshelf_data\\Mystery_Fiction\\raw\\Greenmantle John Buchan.txt\n",
      "2024-12-05 17:55:48,539 - Downloaded: gutenberg_bookshelf_data\\Mystery_Fiction\\raw\\The Works of Edgar Allan Poe — Volume 5 Edgar Allan Poe.txt\n",
      "2024-12-05 17:55:48,993 - Downloaded: gutenberg_bookshelf_data\\Mystery_Fiction\\raw\\Dracula Bram Stoker.txt\n",
      "2024-12-05 17:55:50,451 - Downloaded: gutenberg_bookshelf_data\\Mystery_Fiction\\raw\\The Secret Agent_ A Simple Tale Joseph Conrad.txt\n",
      "2024-12-05 17:55:50,860 - Downloaded: gutenberg_bookshelf_data\\Mystery_Fiction\\raw\\The Works of Edgar Allan Poe — Volume 2 Edgar Allan Poe.txt\n",
      "2024-12-05 17:55:51,349 - Downloaded: gutenberg_bookshelf_data\\Mystery_Fiction\\raw\\The Beetle_ A Mystery Richard Marsh.txt\n",
      "2024-12-05 17:55:52,951 - Downloaded: gutenberg_bookshelf_data\\Mystery_Fiction\\raw\\The Mystery of Edwin Drood Charles Dickens.txt\n",
      "2024-12-05 17:55:54,460 - Downloaded: gutenberg_bookshelf_data\\Mystery_Fiction\\raw\\The Works of Edgar Allan Poe — Volume 3 Edgar Allan Poe.txt\n",
      "2024-12-05 17:55:54,948 - Downloaded: gutenberg_bookshelf_data\\Mystery_Fiction\\raw\\The Woman in White Wilkie Collins.txt\n",
      "2024-12-05 17:55:55,537 - Downloaded: gutenberg_bookshelf_data\\Mystery_Fiction\\raw\\The Works of Edgar Allan Poe — Volume 1 Edgar Allan Poe.txt\n",
      "2024-12-05 17:55:56,050 - Downloaded: gutenberg_bookshelf_data\\Mystery_Fiction\\raw\\The Works of Edgar Allan Poe — Volume 4 Edgar Allan Poe.txt\n",
      "2024-12-05 17:55:56,874 - Downloaded: gutenberg_bookshelf_data\\Mystery_Fiction\\raw\\The Moonstone Wilkie Collins.txt\n",
      "2024-12-05 17:55:57,263 - Downloaded: gutenberg_bookshelf_data\\Mystery_Fiction\\raw\\The Thirty-Nine Steps John Buchan.txt\n",
      "2024-12-05 17:55:57,265 - Metadata saved to gutenberg_bookshelf_data\\Mystery_Fiction/metadata.csv\n",
      "2024-12-05 17:55:58,104 - Processed: Dracula Bram Stoker.txt\n",
      "2024-12-05 17:55:58,667 - Processed: Greenmantle John Buchan.txt\n",
      "2024-12-05 17:55:59,272 - Processed: The Beetle_ A Mystery Richard Marsh.txt\n",
      "2024-12-05 17:56:00,393 - Processed: The Moonstone Wilkie Collins.txt\n",
      "2024-12-05 17:56:00,968 - Processed: The Mystery of Edwin Drood Charles Dickens.txt\n",
      "2024-12-05 17:56:01,520 - Processed: The Secret Agent_ A Simple Tale Joseph Conrad.txt\n",
      "2024-12-05 17:56:01,768 - Processed: The Thirty-Nine Steps John Buchan.txt\n",
      "2024-12-05 17:56:03,133 - Processed: The Woman in White Wilkie Collins.txt\n",
      "2024-12-05 17:56:03,640 - Processed: The Works of Edgar Allan Poe — Volume 1 Edgar Allan Poe.txt\n",
      "2024-12-05 17:56:04,193 - Processed: The Works of Edgar Allan Poe — Volume 2 Edgar Allan Poe.txt\n",
      "2024-12-05 17:56:04,751 - Processed: The Works of Edgar Allan Poe — Volume 3 Edgar Allan Poe.txt\n",
      "2024-12-05 17:56:05,252 - Processed: The Works of Edgar Allan Poe — Volume 4 Edgar Allan Poe.txt\n",
      "2024-12-05 17:56:05,728 - Processed: The Works of Edgar Allan Poe — Volume 5 Edgar Allan Poe.txt\n",
      "2024-12-05 17:56:05,729 - Linguistic features saved to gutenberg_bookshelf_data\\Mystery_Fiction\\cleaned/linguistic_features.csv\n",
      "2024-12-05 17:56:05,731 - Processing genre: Gothic Fiction\n",
      "2024-12-05 17:56:06,020 - Skipping non-English book: Le Horla (French)\n",
      "Guy de Maupassant\n",
      "644 downloads\n",
      "2024-12-05 17:56:06,020 - Found 24 books in the selected bookshelf.\n",
      "2024-12-05 17:56:06,563 - Downloaded: gutenberg_bookshelf_data\\Gothic_Fiction\\raw\\The Monk_ A Romance M. G. Lewis.txt\n",
      "2024-12-05 17:56:06,951 - Downloaded: gutenberg_bookshelf_data\\Gothic_Fiction\\raw\\Nightmare Abbey Thomas Love Peacock.txt\n",
      "2024-12-05 17:56:07,508 - Downloaded: gutenberg_bookshelf_data\\Gothic_Fiction\\raw\\Wuthering Heights Emily Brontë.txt\n",
      "2024-12-05 17:56:07,966 - Downloaded: gutenberg_bookshelf_data\\Gothic_Fiction\\raw\\The Private Memoirs and Confessions of a Justified Sinner James Hogg.txt\n",
      "2024-12-05 17:56:08,302 - Downloaded: gutenberg_bookshelf_data\\Gothic_Fiction\\raw\\The Strange Case of Dr. Jekyll and Mr. Hyde Robert Louis Stevenson.txt\n",
      "2024-12-05 17:56:08,721 - Downloaded: gutenberg_bookshelf_data\\Gothic_Fiction\\raw\\The Lair of the White Worm Bram Stoker.txt\n",
      "2024-12-05 17:56:09,059 - Downloaded: gutenberg_bookshelf_data\\Gothic_Fiction\\raw\\The Yellow Wallpaper Charlotte Perkins Gilman.txt\n",
      "2024-12-05 17:56:09,489 - Downloaded: gutenberg_bookshelf_data\\Gothic_Fiction\\raw\\Wieland; Or, The Transformation_ An American Tale Charles Brockden Brown.txt\n",
      "2024-12-05 17:56:10,253 - Downloaded: gutenberg_bookshelf_data\\Gothic_Fiction\\raw\\The Mysteries of Udolpho Ann Ward Radcliffe.txt\n",
      "2024-12-05 17:56:10,759 - Downloaded: gutenberg_bookshelf_data\\Gothic_Fiction\\raw\\Caleb Williams; Or, Things as They Are William Godwin.txt\n",
      "2024-12-05 17:56:11,120 - Downloaded: gutenberg_bookshelf_data\\Gothic_Fiction\\raw\\The Vampyre; a Tale John William Polidori.txt\n",
      "2024-12-05 17:56:11,606 - Downloaded: gutenberg_bookshelf_data\\Gothic_Fiction\\raw\\The Lock and Key Library_ Classic Mystery and Detective Stories_ Old Time English.txt\n",
      "2024-12-05 17:56:13,360 - Downloaded: gutenberg_bookshelf_data\\Gothic_Fiction\\raw\\Mosses from an old manse Nathaniel Hawthorne.txt\n",
      "2024-12-05 17:56:15,033 - Downloaded: gutenberg_bookshelf_data\\Gothic_Fiction\\raw\\Dracula Bram Stoker.txt\n",
      "2024-12-05 17:56:16,300 - Downloaded: gutenberg_bookshelf_data\\Gothic_Fiction\\raw\\Frankenstein; Or, The Modern Prometheus Mary Wollstonecraft Shelley.txt\n",
      "2024-12-05 17:56:16,643 - Downloaded: gutenberg_bookshelf_data\\Gothic_Fiction\\raw\\Carmilla Joseph Sheridan Le Fanu.txt\n",
      "2024-12-05 17:56:17,528 - Downloaded: gutenberg_bookshelf_data\\Gothic_Fiction\\raw\\The Turn of the Screw Henry James.txt\n",
      "2024-12-05 17:56:18,647 - Downloaded: gutenberg_bookshelf_data\\Gothic_Fiction\\raw\\The Picture of Dorian Gray Oscar Wilde.txt\n",
      "2024-12-05 17:56:19,060 - Downloaded: gutenberg_bookshelf_data\\Gothic_Fiction\\raw\\The Works of Edgar Allan Poe — Volume 2 Edgar Allan Poe.txt\n",
      "2024-12-05 17:56:20,318 - Downloaded: gutenberg_bookshelf_data\\Gothic_Fiction\\raw\\The Phantom of the Opera Gaston Leroux.txt\n",
      "2024-12-05 17:56:20,320 - Metadata saved to gutenberg_bookshelf_data\\Gothic_Fiction/metadata.csv\n",
      "2024-12-05 17:56:21,146 - Processed: Caleb Williams; Or, Things as They Are William Godwin.txt\n",
      "2024-12-05 17:56:21,315 - Processed: Carmilla Joseph Sheridan Le Fanu.txt\n",
      "2024-12-05 17:56:22,175 - Processed: Dracula Bram Stoker.txt\n",
      "2024-12-05 17:56:22,586 - Processed: Frankenstein; Or, The Modern Prometheus Mary Wollstonecraft Shelley.txt\n",
      "2024-12-05 17:56:23,403 - Processed: Mosses from an old manse Nathaniel Hawthorne.txt\n",
      "2024-12-05 17:56:23,574 - Processed: Nightmare Abbey Thomas Love Peacock.txt\n",
      "2024-12-05 17:56:23,946 - Processed: The Lair of the White Worm Bram Stoker.txt\n",
      "2024-12-05 17:56:24,693 - Processed: The Lock and Key Library_ Classic Mystery and Detective Stories_ Old Time English.txt\n",
      "2024-12-05 17:56:24,736 - Processed: The Monkey's Paw W. W. Jacobs.txt\n",
      "2024-12-05 17:56:25,517 - Processed: The Monk_ A Romance M. G. Lewis.txt\n",
      "2024-12-05 17:56:26,966 - Processed: The Mysteries of Udolpho Ann Ward Radcliffe.txt\n",
      "2024-12-05 17:56:27,559 - Processed: The Phantom of the Opera Gaston Leroux.txt\n",
      "2024-12-05 17:56:28,072 - Processed: The Picture of Dorian Gray Oscar Wilde.txt\n",
      "2024-12-05 17:56:28,517 - Processed: The Private Memoirs and Confessions of a Justified Sinner James Hogg.txt\n",
      "2024-12-05 17:56:28,669 - Processed: The Strange Case of Dr. Jekyll and Mr. Hyde Robert Louis Stevenson.txt\n",
      "2024-12-05 17:56:28,907 - Processed: The Turn of the Screw Henry James.txt\n",
      "2024-12-05 17:56:28,994 - Processed: The Vampyre; a Tale John William Polidori.txt\n",
      "2024-12-05 17:56:29,522 - Processed: The Works of Edgar Allan Poe — Volume 2 Edgar Allan Poe.txt\n",
      "2024-12-05 17:56:29,577 - Processed: The Yellow Wallpaper Charlotte Perkins Gilman.txt\n",
      "2024-12-05 17:56:30,081 - Processed: Wieland; Or, The Transformation_ An American Tale Charles Brockden Brown.txt\n",
      "2024-12-05 17:56:30,748 - Processed: Wuthering Heights Emily Brontë.txt\n",
      "2024-12-05 17:56:30,750 - Linguistic features saved to gutenberg_bookshelf_data\\Gothic_Fiction\\cleaned/linguistic_features.csv\n",
      "2024-12-05 17:56:30,753 - Processing genre: Crime Fiction\n",
      "2024-12-05 17:56:30,965 - Found 25 books in the selected bookshelf.\n",
      "2024-12-05 17:56:31,401 - Downloaded: gutenberg_bookshelf_data\\Crime_Fiction\\raw\\The Extraordinary Adventures of Arsène Lupin, Gentleman-Burglar Maurice Leblanc.txt\n",
      "2024-12-05 17:56:31,688 - Failed to download: https://www.gutenberg.org/ebooks/20592.txt.utf-8\n",
      "2024-12-05 17:56:32,794 - Downloaded: gutenberg_bookshelf_data\\Crime_Fiction\\raw\\The Amateur Cracksman E. W. Hornung.txt\n",
      "2024-12-05 17:56:33,842 - Downloaded: gutenberg_bookshelf_data\\Crime_Fiction\\raw\\Arsène Lupin Maurice Leblanc.txt\n",
      "2024-12-05 17:56:34,286 - Downloaded: gutenberg_bookshelf_data\\Crime_Fiction\\raw\\The Eight Strokes of the Clock Maurice Leblanc.txt\n",
      "2024-12-05 17:56:35,646 - Downloaded: gutenberg_bookshelf_data\\Crime_Fiction\\raw\\The Return of Dr. Fu-Manchu Sax Rohmer.txt\n",
      "2024-12-05 17:56:35,924 - Downloaded: gutenberg_bookshelf_data\\Crime_Fiction\\raw\\An African Millionaire_ Episodes in the Life of the Illustrious Colonel Clay Grant Allen.txt\n",
      "2024-12-05 17:56:36,331 - Downloaded: gutenberg_bookshelf_data\\Crime_Fiction\\raw\\Bat Wing Sax Rohmer.txt\n",
      "2024-12-05 17:56:36,738 - Downloaded: gutenberg_bookshelf_data\\Crime_Fiction\\raw\\Brood of the Witch-Queen Sax Rohmer.txt\n",
      "2024-12-05 17:56:37,161 - Downloaded: gutenberg_bookshelf_data\\Crime_Fiction\\raw\\The Devil Doctor Sax Rohmer.txt\n",
      "2024-12-05 17:56:38,597 - Downloaded: gutenberg_bookshelf_data\\Crime_Fiction\\raw\\Dope Sax Rohmer.txt\n",
      "2024-12-05 17:56:40,006 - Downloaded: gutenberg_bookshelf_data\\Crime_Fiction\\raw\\The Teeth of the Tiger Maurice Leblanc.txt\n",
      "2024-12-05 17:56:40,708 - Downloaded: gutenberg_bookshelf_data\\Crime_Fiction\\raw\\Crime and Punishment Fyodor Dostoyevsky.txt\n",
      "2024-12-05 17:56:41,178 - Downloaded: gutenberg_bookshelf_data\\Crime_Fiction\\raw\\Mr. Justice Raffles E. W. Hornung.txt\n",
      "2024-12-05 17:56:41,590 - Downloaded: gutenberg_bookshelf_data\\Crime_Fiction\\raw\\The Crystal Stopper Maurice Leblanc.txt\n",
      "2024-12-05 17:56:41,981 - Downloaded: gutenberg_bookshelf_data\\Crime_Fiction\\raw\\Raffles_ Further Adventures of the Amateur Cracksman E. W. Hornung.txt\n",
      "2024-12-05 17:56:42,372 - Downloaded: gutenberg_bookshelf_data\\Crime_Fiction\\raw\\Dead Men Tell No Tales E. W. Hornung.txt\n",
      "2024-12-05 17:56:42,858 - Downloaded: gutenberg_bookshelf_data\\Crime_Fiction\\raw\\The Yellow Claw Sax Rohmer.txt\n",
      "2024-12-05 17:56:43,297 - Downloaded: gutenberg_bookshelf_data\\Crime_Fiction\\raw\\Fire-Tongue Sax Rohmer.txt\n",
      "2024-12-05 17:56:44,674 - Downloaded: gutenberg_bookshelf_data\\Crime_Fiction\\raw\\The Shadow of the Rope E. W. Hornung.txt\n",
      "2024-12-05 17:56:44,676 - Metadata saved to gutenberg_bookshelf_data\\Crime_Fiction/metadata.csv\n",
      "2024-12-05 17:56:45,058 - Processed: A Thief in the Night_ A Book of Raffles' Adventures E. W. Hornung.txt\n",
      "2024-12-05 17:56:45,077 - Processed: An African Millionaire_ Episodes in the Life of the Illustrious Colonel Clay Grant Allen.txt\n",
      "2024-12-05 17:56:45,565 - Processed: Arsène Lupin Maurice Leblanc.txt\n",
      "2024-12-05 17:56:46,158 - Processed: Bat Wing Sax Rohmer.txt\n",
      "2024-12-05 17:56:46,616 - Processed: Brood of the Witch-Queen Sax Rohmer.txt\n",
      "2024-12-05 17:56:47,930 - Processed: Crime and Punishment Fyodor Dostoyevsky.txt\n",
      "2024-12-05 17:56:48,257 - Processed: Dead Men Tell No Tales E. W. Hornung.txt\n",
      "2024-12-05 17:56:48,817 - Processed: Dope Sax Rohmer.txt\n",
      "2024-12-05 17:56:49,269 - Processed: Fire-Tongue Sax Rohmer.txt\n",
      "2024-12-05 17:56:49,700 - Processed: Mr. Justice Raffles E. W. Hornung.txt\n",
      "2024-12-05 17:56:50,188 - Processed: Murder in the Gunroom H. Beam Piper.txt\n",
      "2024-12-05 17:56:50,501 - Processed: Raffles_ Further Adventures of the Amateur Cracksman E. W. Hornung.txt\n",
      "2024-12-05 17:56:50,859 - Processed: The Amateur Cracksman E. W. Hornung.txt\n",
      "2024-12-05 17:56:51,361 - Processed: The Crystal Stopper Maurice Leblanc.txt\n",
      "2024-12-05 17:56:51,840 - Processed: The Devil Doctor Sax Rohmer.txt\n",
      "2024-12-05 17:56:52,333 - Processed: The Eight Strokes of the Clock Maurice Leblanc.txt\n",
      "2024-12-05 17:56:52,685 - Processed: The Extraordinary Adventures of Arsène Lupin, Gentleman-Burglar Maurice Leblanc.txt\n",
      "2024-12-05 17:56:53,097 - Processed: The Hollow Needle; Further adventures of Arsène Lupin Maurice Leblanc.txt\n",
      "2024-12-05 17:56:53,616 - Processed: The Return of Dr. Fu-Manchu Sax Rohmer.txt\n",
      "2024-12-05 17:56:54,105 - Processed: The Shadow of the Rope E. W. Hornung.txt\n",
      "2024-12-05 17:56:55,012 - Processed: The Teeth of the Tiger Maurice Leblanc.txt\n",
      "2024-12-05 17:56:55,666 - Processed: The Yellow Claw Sax Rohmer.txt\n",
      "2024-12-05 17:56:55,668 - Linguistic features saved to gutenberg_bookshelf_data\\Crime_Fiction\\cleaned/linguistic_features.csv\n",
      "2024-12-05 17:56:55,670 - Processing genre: Romantic Fiction\n",
      "2024-12-05 17:56:55,797 - Found 6 books in the selected bookshelf.\n",
      "2024-12-05 17:56:56,303 - Downloaded: gutenberg_bookshelf_data\\Romantic_Fiction\\raw\\Kun ruusu puhkeaa (Finnish) Louisa May Alcott.txt\n",
      "2024-12-05 17:56:57,864 - Downloaded: gutenberg_bookshelf_data\\Romantic_Fiction\\raw\\Wastralls_ A Novel C. A. Dawson Scott.txt\n",
      "2024-12-05 17:56:58,349 - Downloaded: gutenberg_bookshelf_data\\Romantic_Fiction\\raw\\Kate Vernon_ A Tale. Vol. 1 (of 3) Mrs. Alexander.txt\n",
      "2024-12-05 17:57:00,254 - Downloaded: gutenberg_bookshelf_data\\Romantic_Fiction\\raw\\Only a girl's love Charles Garvice.txt\n",
      "2024-12-05 17:57:00,671 - Downloaded: gutenberg_bookshelf_data\\Romantic_Fiction\\raw\\Kate Vernon_ A Tale. Vol. 2 (of 3) Mrs. Alexander.txt\n",
      "2024-12-05 17:57:01,101 - Downloaded: gutenberg_bookshelf_data\\Romantic_Fiction\\raw\\Star of India Alice Perrin.txt\n",
      "2024-12-05 17:57:01,103 - Metadata saved to gutenberg_bookshelf_data\\Romantic_Fiction/metadata.csv\n",
      "2024-12-05 17:57:01,366 - Processed: Kate Vernon_ A Tale. Vol. 1 (of 3) Mrs. Alexander.txt\n",
      "2024-12-05 17:57:01,660 - Processed: Kate Vernon_ A Tale. Vol. 2 (of 3) Mrs. Alexander.txt\n",
      "2024-12-05 17:57:02,107 - Processed: Kun ruusu puhkeaa (Finnish) Louisa May Alcott.txt\n",
      "2024-12-05 17:57:03,190 - Processed: Only a girl's love Charles Garvice.txt\n",
      "2024-12-05 17:57:03,681 - Processed: Star of India Alice Perrin.txt\n",
      "2024-12-05 17:57:04,324 - Processed: Wastralls_ A Novel C. A. Dawson Scott.txt\n",
      "2024-12-05 17:57:04,326 - Linguistic features saved to gutenberg_bookshelf_data\\Romantic_Fiction\\cleaned/linguistic_features.csv\n",
      "2024-12-05 17:57:04,328 - Processing genre: Horror\n",
      "2024-12-05 17:57:04,486 - Found 25 books in the selected bookshelf.\n",
      "2024-12-05 17:57:05,036 - Downloaded: gutenberg_bookshelf_data\\Horror\\raw\\The Trial Franz Kafka.txt\n",
      "2024-12-05 17:57:05,563 - Downloaded: gutenberg_bookshelf_data\\Horror\\raw\\The Works of Edgar Allan Poe — Volume 2 Edgar Allan Poe.txt\n",
      "2024-12-05 17:57:06,013 - Downloaded: gutenberg_bookshelf_data\\Horror\\raw\\The Works of Edgar Allan Poe — Volume 5 Edgar Allan Poe.txt\n",
      "2024-12-05 17:57:06,580 - Downloaded: gutenberg_bookshelf_data\\Horror\\raw\\The Jewel of Seven Stars Bram Stoker.txt\n",
      "2024-12-05 17:57:06,935 - Downloaded: gutenberg_bookshelf_data\\Horror\\raw\\Metamorphosis Franz Kafka.txt\n",
      "2024-12-05 17:57:07,840 - Downloaded: gutenberg_bookshelf_data\\Horror\\raw\\Varney the Vampire; Or, the Feast of Blood Thomas Peckett Prest and James Malcolm Rymer.txt\n",
      "2024-12-05 17:57:08,261 - Downloaded: gutenberg_bookshelf_data\\Horror\\raw\\Twenty-Five Ghost Stories.txt\n",
      "2024-12-05 17:57:08,505 - Failed to download: https://www.gutenberg.org/ebooks/19797.txt.utf-8\n",
      "2024-12-05 17:57:09,007 - Downloaded: gutenberg_bookshelf_data\\Horror\\raw\\The King in Yellow Robert W. Chambers.txt\n",
      "2024-12-05 17:57:10,087 - Downloaded: gutenberg_bookshelf_data\\Horror\\raw\\Ghost Stories of an Antiquary M. R. James.txt\n",
      "2024-12-05 17:57:10,544 - Downloaded: gutenberg_bookshelf_data\\Horror\\raw\\Dracula's Guest Bram Stoker.txt\n",
      "2024-12-05 17:57:11,069 - Downloaded: gutenberg_bookshelf_data\\Horror\\raw\\The Works of Edgar Allan Poe — Volume 4 Edgar Allan Poe.txt\n",
      "2024-12-05 17:57:11,529 - Downloaded: gutenberg_bookshelf_data\\Horror\\raw\\The Strange Case of Dr. Jekyll and Mr. Hyde Robert Louis Stevenson.txt\n",
      "2024-12-05 17:57:12,931 - Downloaded: gutenberg_bookshelf_data\\Horror\\raw\\The Works of Edgar Allan Poe — Volume 3 Edgar Allan Poe.txt\n",
      "2024-12-05 17:57:13,348 - Downloaded: gutenberg_bookshelf_data\\Horror\\raw\\Carmilla Joseph Sheridan Le Fanu.txt\n",
      "2024-12-05 17:57:13,768 - Downloaded: gutenberg_bookshelf_data\\Horror\\raw\\The Turn of the Screw Henry James.txt\n",
      "2024-12-05 17:57:14,461 - Downloaded: gutenberg_bookshelf_data\\Horror\\raw\\The Works of Edgar Allan Poe — Volume 1 Edgar Allan Poe.txt\n",
      "2024-12-05 17:57:15,155 - Downloaded: gutenberg_bookshelf_data\\Horror\\raw\\The Great God Pan Arthur Machen.txt\n",
      "2024-12-05 17:57:15,468 - Downloaded: gutenberg_bookshelf_data\\Horror\\raw\\The Vampyre; a Tale John William Polidori.txt\n",
      "2024-12-05 17:57:15,757 - Downloaded: gutenberg_bookshelf_data\\Horror\\raw\\An Occurrence at Owl Creek Bridge Ambrose Bierce.txt\n",
      "2024-12-05 17:57:15,759 - Metadata saved to gutenberg_bookshelf_data\\Horror/metadata.csv\n",
      "2024-12-05 17:57:15,808 - Processed: An Occurrence at Owl Creek Bridge Ambrose Bierce.txt\n",
      "2024-12-05 17:57:15,974 - Processed: Carmilla Joseph Sheridan Le Fanu.txt\n",
      "2024-12-05 17:57:16,287 - Processed: Dracula's Guest Bram Stoker.txt\n",
      "2024-12-05 17:57:16,544 - Processed: Ghost Stories of an Antiquary M. R. James.txt\n",
      "2024-12-05 17:57:16,668 - Processed: Metamorphosis Franz Kafka.txt\n",
      "2024-12-05 17:57:16,799 - Processed: The Great God Pan Arthur Machen.txt\n",
      "2024-12-05 17:57:17,302 - Processed: The Jewel of Seven Stars Bram Stoker.txt\n",
      "2024-12-05 17:57:17,772 - Processed: The King in Yellow Robert W. Chambers.txt\n",
      "2024-12-05 17:57:17,927 - Processed: The Strange Case of Dr. Jekyll and Mr. Hyde Robert Louis Stevenson.txt\n",
      "2024-12-05 17:57:18,425 - Processed: The Trial Franz Kafka.txt\n",
      "2024-12-05 17:57:18,679 - Processed: The Turn of the Screw Henry James.txt\n",
      "2024-12-05 17:57:18,758 - Processed: The Vampyre; a Tale John William Polidori.txt\n",
      "2024-12-05 17:57:19,251 - Processed: The Works of Edgar Allan Poe — Volume 1 Edgar Allan Poe.txt\n",
      "2024-12-05 17:57:19,762 - Processed: The Works of Edgar Allan Poe — Volume 2 Edgar Allan Poe.txt\n",
      "2024-12-05 17:57:20,273 - Processed: The Works of Edgar Allan Poe — Volume 3 Edgar Allan Poe.txt\n",
      "2024-12-05 17:57:20,766 - Processed: The Works of Edgar Allan Poe — Volume 4 Edgar Allan Poe.txt\n",
      "2024-12-05 17:57:21,172 - Processed: The Works of Edgar Allan Poe — Volume 5 Edgar Allan Poe.txt\n",
      "2024-12-05 17:57:21,484 - Processed: Twenty-Five Ghost Stories.txt\n",
      "2024-12-05 17:57:23,462 - Processed: Varney the Vampire; Or, the Feast of Blood Thomas Peckett Prest and James Malcolm Rymer.txt\n",
      "2024-12-05 17:57:23,464 - Linguistic features saved to gutenberg_bookshelf_data\\Horror\\cleaned/linguistic_features.csv\n",
      "2024-12-05 17:57:23,469 - Processing genre: Detective Fiction\n",
      "2024-12-05 17:57:23,656 - Found 25 books in the selected bookshelf.\n",
      "2024-12-05 17:57:25,373 - Downloaded: gutenberg_bookshelf_data\\Detective_Fiction\\raw\\The Works of Edgar Allan Poe — Volume 2 Edgar Allan Poe.txt\n",
      "2024-12-05 17:57:26,588 - Downloaded: gutenberg_bookshelf_data\\Detective_Fiction\\raw\\Whose Body_ A Lord Peter Wimsey Novel Dorothy L. Sayers.txt\n",
      "2024-12-05 17:57:27,694 - Downloaded: gutenberg_bookshelf_data\\Detective_Fiction\\raw\\His Last Bow_ An Epilogue of Sherlock Holmes Arthur Conan Doyle.txt\n",
      "2024-12-05 17:57:28,089 - Downloaded: gutenberg_bookshelf_data\\Detective_Fiction\\raw\\Poirot Investigates Agatha Christie.txt\n",
      "2024-12-05 17:57:28,675 - Downloaded: gutenberg_bookshelf_data\\Detective_Fiction\\raw\\The wisdom of Father Brown G. K. Chesterton.txt\n",
      "2024-12-05 17:57:29,189 - Downloaded: gutenberg_bookshelf_data\\Detective_Fiction\\raw\\The Mysterious Affair at Styles Agatha Christie.txt\n",
      "2024-12-05 17:57:29,611 - Downloaded: gutenberg_bookshelf_data\\Detective_Fiction\\raw\\The Adventures of Sherlock Holmes Arthur Conan Doyle.txt\n",
      "2024-12-05 17:57:29,930 - Downloaded: gutenberg_bookshelf_data\\Detective_Fiction\\raw\\Hunted Down_ The Detective Stories of Charles Dickens Charles Dickens.txt\n",
      "2024-12-05 17:57:30,297 - Failed to download: https://www.gutenberg.org/ebooks/9551.txt.utf-8\n",
      "2024-12-05 17:57:30,692 - Downloaded: gutenberg_bookshelf_data\\Detective_Fiction\\raw\\The Murder on the Links Agatha Christie.txt\n",
      "2024-12-05 17:57:32,476 - Downloaded: gutenberg_bookshelf_data\\Detective_Fiction\\raw\\The Memoirs of Sherlock Holmes Arthur Conan Doyle.txt\n",
      "2024-12-05 17:57:34,193 - Downloaded: gutenberg_bookshelf_data\\Detective_Fiction\\raw\\The Return of Sherlock Holmes Arthur Conan Doyle.txt\n",
      "2024-12-05 17:57:34,766 - Downloaded: gutenberg_bookshelf_data\\Detective_Fiction\\raw\\The Return of Sherlock Holmes Arthur Conan Doyle.txt\n",
      "2024-12-05 17:57:35,216 - Downloaded: gutenberg_bookshelf_data\\Detective_Fiction\\raw\\The Red House Mystery A. A. Milne.txt\n",
      "2024-12-05 17:57:35,701 - Downloaded: gutenberg_bookshelf_data\\Detective_Fiction\\raw\\The Sign of the Four Arthur Conan Doyle.txt\n",
      "2024-12-05 17:57:36,102 - Downloaded: gutenberg_bookshelf_data\\Detective_Fiction\\raw\\The Hound of the Baskervilles Arthur Conan Doyle.txt\n",
      "2024-12-05 17:57:37,532 - Downloaded: gutenberg_bookshelf_data\\Detective_Fiction\\raw\\The Secret Adversary Agatha Christie.txt\n",
      "2024-12-05 17:57:39,638 - Downloaded: gutenberg_bookshelf_data\\Detective_Fiction\\raw\\The Moonstone Wilkie Collins.txt\n",
      "2024-12-05 17:57:40,055 - Downloaded: gutenberg_bookshelf_data\\Detective_Fiction\\raw\\The innocence of Father Brown G. K. Chesterton.txt\n",
      "2024-12-05 17:57:41,938 - Downloaded: gutenberg_bookshelf_data\\Detective_Fiction\\raw\\The Man in the Brown Suit Agatha Christie.txt\n",
      "2024-12-05 17:57:41,940 - Metadata saved to gutenberg_bookshelf_data\\Detective_Fiction/metadata.csv\n",
      "2024-12-05 17:57:42,306 - Processed: His Last Bow_ An Epilogue of Sherlock Holmes Arthur Conan Doyle.txt\n",
      "2024-12-05 17:57:42,378 - Processed: Hunted Down_ The Detective Stories of Charles Dickens Charles Dickens.txt\n",
      "2024-12-05 17:57:42,744 - Processed: Poirot Investigates Agatha Christie.txt\n",
      "2024-12-05 17:57:43,333 - Processed: The Adventures of Sherlock Holmes Arthur Conan Doyle.txt\n",
      "2024-12-05 17:57:43,670 - Processed: The Hound of the Baskervilles Arthur Conan Doyle.txt\n",
      "2024-12-05 17:57:44,110 - Processed: The innocence of Father Brown G. K. Chesterton.txt\n",
      "2024-12-05 17:57:44,618 - Processed: The Man in the Brown Suit Agatha Christie.txt\n",
      "2024-12-05 17:57:45,153 - Processed: The Memoirs of Sherlock Holmes Arthur Conan Doyle.txt\n",
      "2024-12-05 17:57:46,323 - Processed: The Moonstone Wilkie Collins.txt\n",
      "2024-12-05 17:57:46,770 - Processed: The Murder on the Links Agatha Christie.txt\n",
      "2024-12-05 17:57:47,160 - Processed: The Mysterious Affair at Styles Agatha Christie.txt\n",
      "2024-12-05 17:57:47,653 - Processed: The Red House Mystery A. A. Milne.txt\n",
      "2024-12-05 17:57:48,312 - Processed: The Return of Sherlock Holmes Arthur Conan Doyle.txt\n",
      "2024-12-05 17:57:48,878 - Processed: The Secret Adversary Agatha Christie.txt\n",
      "2024-12-05 17:57:49,140 - Processed: The Sign of the Four Arthur Conan Doyle.txt\n",
      "2024-12-05 17:57:49,563 - Processed: The wisdom of Father Brown G. K. Chesterton.txt\n",
      "2024-12-05 17:57:50,073 - Processed: The Works of Edgar Allan Poe — Volume 2 Edgar Allan Poe.txt\n",
      "2024-12-05 17:57:50,449 - Processed: Whose Body_ A Lord Peter Wimsey Novel Dorothy L. Sayers.txt\n",
      "2024-12-05 17:57:50,452 - Linguistic features saved to gutenberg_bookshelf_data\\Detective_Fiction\\cleaned/linguistic_features.csv\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import re\n",
    "import random\n",
    "import requests\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup\n",
    "from nltk.tokenize import sent_tokenize, word_tokenize\n",
    "from collections import Counter\n",
    "import logging\n",
    "\n",
    "\n",
    "# Setup logging\n",
    "logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s')\n",
    "logger = logging.getLogger(__name__)\n",
    "\n",
    "BASE_URL = \"https://www.gutenberg.org\"\n",
    "\n",
    "def get_bookshelves():\n",
    "    \"\"\"\n",
    "    Fetches all bookshelves (genres) from Project Gutenberg.\n",
    "    \"\"\"\n",
    "    response = requests.get(f\"{BASE_URL}/ebooks/bookshelf/\")\n",
    "    if response.status_code != 200:\n",
    "        raise Exception(\"Failed to fetch the bookshelves page.\")\n",
    "\n",
    "    soup = BeautifulSoup(response.text, 'html.parser')\n",
    "    bookshelves = {}\n",
    "    for link in soup.find_all(\"a\", href=True):\n",
    "        if \"/ebooks/bookshelf/\" in link['href']:\n",
    "            genre_name = link.text.strip()\n",
    "            genre_url = BASE_URL + link['href']\n",
    "            bookshelves[genre_name] = genre_url\n",
    "\n",
    "    logger.info(f\"Found {len(bookshelves)} bookshelves.\")\n",
    "    return bookshelves\n",
    "\n",
    "def fetch_books_from_shelf(shelf_url, output_folder, num_books=10):\n",
    "    \"\"\"\n",
    "    Fetches books from a specific bookshelf URL, removes non-English books, \n",
    "    and cleans the title to exclude download numbers or extra details.\n",
    "    \"\"\"\n",
    "    response = requests.get(shelf_url)\n",
    "    if response.status_code != 200:\n",
    "        raise Exception(f\"Failed to fetch books from bookshelf: {shelf_url}\")\n",
    "\n",
    "    soup = BeautifulSoup(response.text, 'html.parser')\n",
    "    book_links = []\n",
    "    seen_urls = set()\n",
    "\n",
    "    # Extract book links\n",
    "    for link in soup.find_all(\"a\", href=True):\n",
    "        if link['href'].startswith(\"/ebooks/\") and link['href'].split(\"/\")[-1].isdigit():\n",
    "            full_url = f\"{BASE_URL}{link['href']}.txt.utf-8\"\n",
    "            if full_url not in seen_urls:\n",
    "                seen_urls.add(full_url)\n",
    "                raw_title = link.text.strip()\n",
    "                \n",
    "                # Remove downloads information and filter non-English books\n",
    "                if \"(French)\" in raw_title or \"(German)\" in raw_title or \"(Spanish)\" in raw_title:\n",
    "                    logger.info(f\"Skipping non-English book: {raw_title}\")\n",
    "                    continue\n",
    "\n",
    "                # Clean title to remove extra details like downloads\n",
    "                sanitized_title = re.sub(r\"\\s*\\d+\\s*downloads$\", \"\", raw_title).strip()\n",
    "                sanitized_title = re.sub(r'[\\\\/*?:\"<>|]', \"_\", sanitized_title)\n",
    "                sanitized_title = re.sub(r'\\s+', ' ', sanitized_title).strip()  # Remove extra spaces\n",
    "                \n",
    "                book_links.append({\n",
    "                    \"title\": sanitized_title,\n",
    "                    \"url\": full_url\n",
    "                })\n",
    "\n",
    "    logger.info(f\"Found {len(book_links)} books in the selected bookshelf.\")\n",
    "    if len(book_links) == 0:\n",
    "        logger.warning(\"No books found in this bookshelf.\")\n",
    "        return []\n",
    "\n",
    "    # Select a limited number of books\n",
    "    selected_books = random.sample(book_links, min(num_books, len(book_links)))\n",
    "\n",
    "    # Prepare output folder\n",
    "    os.makedirs(output_folder, exist_ok=True)\n",
    "    raw_folder = os.path.join(output_folder, \"raw\")\n",
    "    os.makedirs(raw_folder, exist_ok=True)\n",
    "\n",
    "    metadata = []\n",
    "    for book in selected_books:\n",
    "        title = book[\"title\"]\n",
    "        raw_file_path = os.path.join(raw_folder, f\"{title}.txt\")\n",
    "        if download_gutenberg_text(book[\"url\"], raw_file_path):\n",
    "            metadata.append({\"title\": book[\"title\"], \"url\": book[\"url\"], \"genre\": os.path.basename(shelf_url)})\n",
    "\n",
    "    # Save metadata\n",
    "    if metadata:\n",
    "        metadata_df = pd.DataFrame(metadata)\n",
    "        metadata_df.to_csv(os.path.join(output_folder, \"metadata.csv\"), index=False)\n",
    "        logger.info(f\"Metadata saved to {output_folder}/metadata.csv\")\n",
    "    return metadata\n",
    "\n",
    "\n",
    "\n",
    "def download_gutenberg_text(url, save_path):\n",
    "    \"\"\"\n",
    "    Downloads text from a Project Gutenberg URL and saves it locally.\n",
    "    \"\"\"\n",
    "    response = requests.get(url)\n",
    "    if response.status_code == 200:\n",
    "        with open(save_path, 'w', encoding='utf-8') as file:\n",
    "            file.write(response.text)\n",
    "        logger.info(f\"Downloaded: {save_path}\")\n",
    "        return True\n",
    "    else:\n",
    "        logger.error(f\"Failed to download: {url}\")\n",
    "        return False\n",
    "\n",
    "def preprocess_books(raw_folder, clean_folder):\n",
    "    \"\"\"\n",
    "    Preprocesses all raw text files: cleans the text and extracts linguistic features.\n",
    "    \"\"\"\n",
    "    os.makedirs(clean_folder, exist_ok=True)\n",
    "    features = []\n",
    "\n",
    "    for file_name in os.listdir(raw_folder):\n",
    "        if not file_name.endswith(\".txt\"):\n",
    "            continue\n",
    "\n",
    "        raw_file_path = os.path.join(raw_folder, file_name)\n",
    "        with open(raw_file_path, 'r', encoding='utf-8') as raw_file:\n",
    "            raw_text = raw_file.read()\n",
    "\n",
    "        # Clean text\n",
    "        cleaned_text = clean_text(raw_text)\n",
    "        clean_file_path = os.path.join(clean_folder, file_name)\n",
    "        with open(clean_file_path, 'w', encoding='utf-8') as clean_file:\n",
    "            clean_file.write(cleaned_text)\n",
    "        \n",
    "        # Extract linguistic features\n",
    "        tokens = word_tokenize(cleaned_text)\n",
    "        sentences = sent_tokenize(cleaned_text)\n",
    "        word_freq = Counter(tokens)\n",
    "        vocab_richness = len(set(tokens)) / len(tokens) if tokens else 0\n",
    "        avg_sentence_length = sum(len(word_tokenize(s)) for s in sentences) / len(sentences) if sentences else 0\n",
    "\n",
    "        # Store features\n",
    "        features.append({\n",
    "            \"file_name\": file_name,\n",
    "            \"vocab_richness\": vocab_richness,\n",
    "            \"avg_sentence_length\": avg_sentence_length,\n",
    "            \"num_sentences\": len(sentences),\n",
    "            \"num_words\": len(tokens),\n",
    "            \"most_common_word\": word_freq.most_common(1)[0][0] if word_freq else None\n",
    "        })\n",
    "        logger.info(f\"Processed: {file_name}\")\n",
    "\n",
    "    # Save features to CSV\n",
    "    if features:\n",
    "        features_df = pd.DataFrame(features)\n",
    "        features_df.to_csv(os.path.join(clean_folder, \"linguistic_features.csv\"), index=False)\n",
    "        logger.info(f\"Linguistic features saved to {clean_folder}/linguistic_features.csv\")\n",
    "    else:\n",
    "        logger.warning(\"No features extracted; the dataset might be empty.\")\n",
    "\n",
    "def clean_text(text):\n",
    "    \"\"\"\n",
    "    Cleans the raw text by removing headers, footers, and extra spaces.\n",
    "    \"\"\"\n",
    "    text = re.sub(r\"\\s+\", \" \", text)  # Remove extra whitespace\n",
    "    start_index = text.find(\"*** START OF THIS PROJECT GUTENBERG EBOOK\")\n",
    "    end_index = text.find(\"*** END OF THIS PROJECT GUTENBERG EBOOK\")\n",
    "    if start_index != -1 and end_index != -1:\n",
    "        text = text[start_index + len(\"*** START OF THIS PROJECT GUTENBERG EBOOK\"):end_index]\n",
    "    return text.strip()\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    # Get bookshelves\n",
    "    bookshelves = get_bookshelves()\n",
    "\n",
    "    # Choose genres to process\n",
    "    selected_genres = [\"Science Fiction\", \"Fantasy\", \"Mystery Fiction\", \"Gothic Fiction\", \"Crime Fiction\", \"Romantic Fiction\", \"Horror\", \"Detective Fiction\"]\n",
    "    output_folder = \"gutenberg_bookshelf_data\"\n",
    "\n",
    "    for genre in selected_genres:\n",
    "        if genre in bookshelves:\n",
    "            logger.info(f\"Processing genre: {genre}\")\n",
    "            shelf_url = bookshelves[genre]\n",
    "            genre_folder = os.path.join(output_folder, genre.replace(\" \", \"_\"))\n",
    "            metadata = fetch_books_from_shelf(shelf_url, genre_folder, num_books=20)\n",
    "            if metadata:\n",
    "                preprocess_books(os.path.join(genre_folder, \"raw\"), os.path.join(genre_folder, \"cleaned\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}