Fix asset extraction bugs. Build script, handle bundle paths. Bump to…

… alpha 0.01a

.gitignore

@@ -10,5 +10,13 @@ __pycache__/
 /chromium
 /chromium-profile
 /cache
+/saves
+/assets
 /xml/gz/v855098/*.xml
-/xml/gz/v855098/*.amf
+/xml/gz/v855098/*.amf
+/xml/gz/v855098/*.json
+/xml/gz/v855038/*.json
+/embeds/Flash/v295145-295127/
+/embeds/Flash/v295765-295762/
+/xml/v295145-295127/
+/xml/v295765-295762/

LINUX.md

@@ -6,6 +6,7 @@ You will need:
 
 - A browser with Flash support (see [Flash Chromium browser](#flash-chromium-browser) section).
 - Adobe Flash Player installed (see [Flash Player for Linux Pepper](#flash-player-for-linux-pepper) section).
+- Assets are not provided. WARC files need to be [downloaded](https://archive.org/details/original-farmville) from the Web Archive, [extracted](https://github.com/recrm/ArchiveTools/blob/master/warc_extractor/warc_extractor.py) and placed inside an `assets/` folder.
 
 ## Flash Chromium browser
 

assets.py

@@ -1,17 +1,17 @@
 import os
+import sys
 import shutil
 import threading
 import requests
 import tqdm
 import hashlib
 
-from bundle import ASSETS_DIR
+from bundle import ASSETS_DIR, TMP_DIR
 from warc_extractor.warc_extractor import _main_interface
 
-WARC_PATH = os.path.join(ASSETS_DIR, "warc")
-EXTRACTION_PATH = os.path.join(ASSETS_DIR, "extracted")
-EXTRACTED_INNER_ASSETS_PATH = os.path.join(EXTRACTION_PATH, "/zynga1-a.akamaihd.net/farmville/assets/hashed/assets")
-INNER_ASSETS_PATH = os.path.join(ASSETS_DIR, "assets")
+WARC_PATH = os.path.join(TMP_DIR, "warc")
+EXTRACTION_PATH = os.path.join(TMP_DIR, "extracted")
+EXTRACTED_INNER_ASSETS_PATH = os.path.join(EXTRACTION_PATH, "zynga1-a.akamaihd.net", "farmville", "assets", "hashed", "assets")
 
 BASE_URL = "https://archive.org/download/original-farmville/"
 FILES = [
@@ -29,17 +29,29 @@
     },
 ]
 
+def __inner_assets_check():
+    return os.path.exists(os.path.join(ASSETS_DIR, "Environment")) and os.path.exists(os.path.join(ASSETS_DIR, "xpromoSupport")) # TODO: check for more directories
+
 def check_assets():
 
     # check if extracted assets directory exists
-    if os.path.exists(INNER_ASSETS_PATH):
-       print(" * Assets directory found.")
-       return
+    if __inner_assets_check():
+        print(" * Assets directory found.")
+        return
+    print(" * Assets not found or missing assets.")
+
+    # check if assets dir already exists and is not empty
+    if os.path.exists(ASSETS_DIR) and os.listdir(ASSETS_DIR):
+        print(" [!] Assets directory found (not empty) but seems incomplete. Remove before proceeding. Exiting...")
+        sys.exit(1)
+    elif os.path.exists(ASSETS_DIR):
+        print(" [!] Empty assets directory found. Removing...")
+        os.rmdir(ASSETS_DIR)
 
-    # check if assets directory exists
-    if not os.path.exists(ASSETS_DIR):
-        print(" * Assets directory not found. Creating assets directory...")
-        os.mkdir(ASSETS_DIR)
+    # check if tmp directory exists
+    if not os.path.exists(TMP_DIR):
+        print(" * Creating tmp directory...")
+        os.mkdir(TMP_DIR)
 
     # check if WARC directory exists and is not empty
     if not os.path.exists(WARC_PATH) or not os.listdir(WARC_PATH):
@@ -70,20 +82,42 @@ def check_assets():
                     continue
                 else:
                     print(" [!] File hash mismatch. Need to redownload. Exiting...")
-                    exit(1)
+                    sys.exit(1)
 
     if missing_warc_files:
         print(" [!] (Re)Downloading missing or corrupted WARC assets file(s)...")
         warc_download(missing_warc_files)
         check_assets()
         return
 
-    # check if extracted assets directory exists and is not empty
-    if not os.path.exists(EXTRACTION_PATH) or not os.listdir(EXTRACTION_PATH):
-        print(" * Extracted assets directory not found. Extracting assets... (one time only, might take a while)")
-        assets_extract()
-        check_assets()
-        return
+    # At this point, assets are not found or incomplete, so we need to extract them
+    print(" * Extracting assets... (one time only, might take a while: up to 1h)")
+    assets_extract()
+
+    # move extracted assets to assets directory
+    print(" * Moving extracted assets to assets directory...")
+
+    # If the destination is a directory or a symlink to a directory, the source is moved inside the directory.
+    # The destination path must not already exist.
+    assert not os.path.exists(ASSETS_DIR)
+
+    # If the destination is on our current filesystem (it should be),
+    # then rename() is used (which should be really efficient), instead of a copy and remove.
+    dest = shutil.move(EXTRACTED_INNER_ASSETS_PATH, ASSETS_DIR)
+    if not dest:
+        print(" [!] Moving extracted assets failed. Exiting...")
+        sys.exit(1)
+
+    # delete tmp extracted assets directory and warc directory
+    print(" * Cleaning up tmp dir...")
+    shutil.rmtree(TMP_DIR)
+    print(" * Assets extracted!")
+
+    # Let's check again if the assets are extracted
+    if not __inner_assets_check():
+        print(" [!] Asset extraction failed. Exiting...")
+        sys.exit(1)
+    return
 
 def warc_download(files_list):
     # create WARC directory
@@ -147,16 +181,5 @@ def assets_extract():
         silence=False
     )
     if not os.path.exists(EXTRACTED_INNER_ASSETS_PATH) or not os.listdir(EXTRACTED_INNER_ASSETS_PATH):
-        print(" [!] Extraction failed. Exiting...")
-        exit(1)
-
-    # move extracted assets to assets directory
-    dest = shutil.move(EXTRACTED_INNER_ASSETS_PATH, INNER_ASSETS_PATH)
-    if not dest:
-        print(" [!] Moving extracted assets failed. Exiting...")
-        exit(1)
-
-    # delete tmp extracted assets directory and warc directory
-    shutil.rmtree(EXTRACTION_PATH)
-    shutil.rmtree(WARC_PATH)
-    print(" * Assets extracted!")
+        print(" [!] Extraction failed. Consider extracting and placing the assets manually. Exiting...")
+        sys.exit(1)

build/build_bundle.bat

@@ -0,0 +1,36 @@
+@echo off
+set NAME=farmvillagers_0.01a
+
+REM remember to activate the venv first (Scripts\activate)
+
+:main
+call :pyInstaller
+echo.
+pause>NUL
+exit
+
+:pyInstaller
+echo [+] Starting pyInstaller...
+pyinstaller ^
+ --onedir ^
+ --console ^
+ --noupx ^
+ --noconfirm ^
+ --paths ..\. ^
+ --add-data "..\\..\\patched;patched" ^
+ --add-data "..\\..\\templates;templates" ^
+ --add-data "..\\..\\villages;villages" ^
+ --add-data "..\\..\\xml;xml" ^
+ --add-data "..\\..\\embeds;embeds" ^
+ --add-data "..\\..\\assethash;assethash" ^
+ --workpath ".\\work" ^
+ --distpath ".\\dist" ^
+ --specpath ".\\spec" ^
+ --contents-directory "bundle" ^
+ --hidden-import cpyamf ^
+ --hidden-import pyamf.amf0 ^
+ --hidden-import pyamf.amf3 ^
+ --icon=..\icon.ico ^
+ --name %NAME% ..\server.py
+echo [+] pyInstaller Done.
+EXIT /B 0

bundle.py

@@ -1,23 +1,37 @@
 import sys
 import os
 
-# Bundled data (extracted to a temp dir)
 
-TMP_BUNDLED_DIR = sys._MEIPASS if getattr(sys, 'frozen', None) else "."
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+TMP_BUNDLED_DIR = BASE_DIR
+
+if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'):
+    print(" [+] Running in bundle/packaged mode")
+    TMP_BUNDLED_DIR = sys._MEIPASS
+    # PyInstaller broke the __file__ attribute of the entry-point script which
+    # used to point to the basename (where the EXE was located). Now it points
+    # to the module's full path within the bundle directory. We need to point
+    # it to the parent directory of the bundle, not the bundle itself.
+    # Source: https://pyinstaller.org/en/stable/runtime-information.html
+    # This change is relevant because Flask uses the __file__ attribute to find
+    # the root directory of the application, and thus assets returned by functions
+    # such as send_from_directory would include the bundle (or _internal) path.
+    # We'll set Flask app.root_path to BASE_DIR instead of the default __file__ value
+    # as a workaround.
+    BASE_DIR = os.path.dirname(BASE_DIR)
+
+# Bundled data (extracted to a temp dir)
 
-ASSETS_DIR = os.path.join(TMP_BUNDLED_DIR, "assets")
-INNER_ASSETS_DIR = os.path.join(ASSETS_DIR, "zynga1-a.akamaihd.net/farmville/assets/hashed/assets")
-EMBEDS_DIR = os.path.join(TMP_BUNDLED_DIR, "embeds")
-ASSETHASH_DIR = os.path.join(TMP_BUNDLED_DIR, "assethash")
-STUB_ASSETS_DIR = os.path.join(TMP_BUNDLED_DIR, "stub")
 PATCHED_ASSETS_DIR = os.path.join(TMP_BUNDLED_DIR, "patched")
 TEMPLATES_DIR = os.path.join(TMP_BUNDLED_DIR, "templates")
-XML_DIR = os.path.join(TMP_BUNDLED_DIR, "xml")
 VILLAGES_DIR = os.path.join(TMP_BUNDLED_DIR, "villages")
-CACHE_DIR = os.path.join(TMP_BUNDLED_DIR, "cache")
+XML_DIR = os.path.join(TMP_BUNDLED_DIR, "xml")
+EMBEDS_DIR = os.path.join(TMP_BUNDLED_DIR, "embeds")
+ASSETHASH_DIR = os.path.join(TMP_BUNDLED_DIR, "assethash")
 
 # Not bundled data (next to server EXE)
 
-BASE_DIR = "."
-
 SAVES_DIR = os.path.join(BASE_DIR, "saves")
+ASSETS_DIR = os.path.join(BASE_DIR, "assets")
+CACHE_DIR = os.path.join(BASE_DIR, "cache")
+TMP_DIR = os.path.join(BASE_DIR, "tmp")

player.py

@@ -1,4 +1,5 @@
 import os
+import sys
 import json
 import copy
 import uuid
@@ -40,14 +41,14 @@ def load_saves() -> None:
     # Saves dir check
     if not os.path.exists(SAVES_DIR):
         try:
-            print(f"Creating '{SAVES_DIR}' folder...")
+            print(f" * Creating '{SAVES_DIR}' folder...")
             os.mkdir(SAVES_DIR)
         except:
-            print(f"Could not create '{SAVES_DIR}' folder.")
-            exit(1)
+            print(f"[!] Could not create '{SAVES_DIR}' folder.")
+            sys.exit(1)
     if not os.path.isdir(SAVES_DIR):
-        print(f"'{SAVES_DIR}' is not a folder... Move the file somewhere else.")
-        exit(1)
+        print(f"[!] '{SAVES_DIR}' is not a folder... Move the file somewhere else.")
+        sys.exit(1)
 
     # Saves in /saves
     for file in os.listdir(SAVES_DIR):

requirements.txt

@@ -1,3 +1,4 @@
+requests==2.28.1
 Flask==2.1.2
 Py3AMF==0.8.10
 tqdm==4.66.5

server.py

@@ -1,5 +1,6 @@
 print (" [+] Loading basics...")
 import os
+import sys
 import json
 
 if os.name == 'nt':
@@ -37,7 +38,7 @@
 import commands
 from engine import timestamp_now
 from version import version_name
-from bundle import ASSETS_DIR, EMBEDS_DIR, ASSETHASH_DIR, STUB_ASSETS_DIR, PATCHED_ASSETS_DIR, TEMPLATES_DIR, XML_DIR
+from bundle import BASE_DIR, ASSETS_DIR, EMBEDS_DIR, ASSETHASH_DIR, PATCHED_ASSETS_DIR, TEMPLATES_DIR, XML_DIR
 from player import save_session
 
 BIND_IP = "127.0.0.1"
@@ -315,5 +316,8 @@ def sn_app_url_gifts():
 print (" [+] Running server...")
 
 if __name__ == '__main__':
-    app.secret_key = 'SECRET_KEY'
+    app.secret_key = os.urandom(24)
+    app.root_path = BASE_DIR
+    app.template_folder = TEMPLATES_DIR
+    app.static_folder = TEMPLATES_DIR
     app.run(host=BIND_IP, port=BIND_PORT, debug=False, threaded=True)

version.py

@@ -1,6 +1,6 @@
 
-version_code = "0.01a.2024_09_16"
-version_name = "pre-alpha " + version_code
+version_code = "0.01a"
+version_name = "alpha " + version_code
 
 def migrate_loaded_save(save: dict):
 

warc_extractor/warc_extractor.py

@@ -690,7 +690,7 @@ def warc_records(string, path):
     """Iterates over warc records in path."""
     for filename in os.listdir(path):
         if re.search(string, filename) and ".warc" in filename:
-            print("parsing", filename)
+            print(" * parsing", filename)
             with WARCFile(path + filename) as warc_file:
                 for record in warc_file:
                     yield record