Upload folder using huggingface_hub

copy_chromadb.py

@@ -83,6 +83,11 @@ db_config = {
         "source_collection_name": "sri_vachana_bhushanam",
         "destination_collection_name": "sri_vachana_bhushanam",
     },
+    "desika_prabandham": {
+        "source_db_path": "../desika_prabandham_ai/chromadb_store",
+        "source_collection_name": "desika_prabandham",
+        "destination_collection_name": "desika_prabandham",
+    },
 }
 
 parser = argparse.ArgumentParser(description="My app with database parameter")

modules/config/__init__.py

@@ -2,6 +2,7 @@ from modules.config.bhagavat_gita import bhagavat_gita_config
 from modules.config.bhagavata_purana import bhagavata_purana_config
 from modules.config.brahma_sutra import brahma_sutra_config
 from modules.config.chathusloki import chathusloki_config
+from modules.config.desika_prabandham import desika_prabandham_config
 from modules.config.divya_prabandham import divya_prabandham_config
 from modules.config.divya_prabandham_taniyans import divya_prabandham_taniyans_config
 from modules.config.kamba_ramayanam import kamba_ramayanam_config
@@ -44,5 +45,6 @@ scripture_configurations = [
     vishnu_sahasranamam_config,
     yt_metadata_config,
     brahma_sutra_config,
-    sri_vachana_bhushanam_config
+    sri_vachana_bhushanam_config,
+    desika_prabandham_config
 ]

modules/config/desika_prabandham.py

@@ -0,0 +1,191 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+
+def fn_list_of_chapters():
+    return [
+        {
+            "chapter_number": 1,
+            "chapter_name": "அமிருதரஞ்சனி",
+            "total_verses": 39,
+        },
+        {
+            "chapter_number": 2,
+            "chapter_name": "அதிகாரசங்கிரகம்",
+            "total_verses": 56,
+        },
+        {
+            "chapter_number": 3,
+            "chapter_name": "அமிருதசுவாதினி",
+            "total_verses": 37,
+        },
+        {
+            "chapter_number": 4,
+            "chapter_name": "பரமபதசோபானம்",
+            "total_verses": 21,
+        },
+        {
+            "chapter_number": 5,
+            "chapter_name": "பரமதபங்கம்",
+            "total_verses": 54,
+        },
+        {
+            "chapter_number": 6,
+            "chapter_name": "மெய்விரதமான்மியம்",
+            "total_verses": 29,
+        },
+        {
+            "chapter_number": 7,
+            "chapter_name": "அடைக்கலப் பத்து",
+            "total_verses": 11,
+        },
+        {
+            "chapter_number": 8,
+            "chapter_name": "அருத்தபஞ்சகம்",
+            "total_verses": 11,
+        },
+        {
+            "chapter_number": 9,
+            "chapter_name": "ஸ்ரீ வைணவதினசரி",
+            "total_verses": 10,
+        },
+        {
+            "chapter_number": 10,
+            "chapter_name": "திருச்சின்ன மாலை",
+            "total_verses": 11,
+        },
+        {
+            "chapter_number": 11,
+            "chapter_name": "பன்னிரு நாமம்",
+            "total_verses": 13,
+        },
+        {
+            "chapter_number": 12,
+            "chapter_name": "திருமந்திரச்சுருக்கு",
+            "total_verses": 10,
+        },
+        {
+            "chapter_number": 13,
+            "chapter_name": "துவயச்சுருக்கு",
+            "total_verses": 12,
+        },
+        {
+            "chapter_number": 14,
+            "chapter_name": "சரமசுலோகச்சுருக்கு",
+            "total_verses": 11,
+        },
+        {
+            "chapter_number": 15,
+            "chapter_name": "கீதார்த்தசங்கிரகம்",
+            "total_verses": 21,
+        },
+        {
+            "chapter_number": 16,
+            "chapter_name": "மும்மணிக்கோவை",
+            "total_verses": 10,
+        },
+        {
+            "chapter_number": 17,
+            "chapter_name": "நவமணிமாலை",
+            "total_verses": 10,
+        },
+        {
+            "chapter_number": 18,
+            "chapter_name": "பிரபந்தசாரம்",
+            "total_verses": 17,
+        },
+        {
+            "chapter_number": 19,
+            "chapter_name": "இந்த ப்ரபந்தவரலாறு",
+            "total_verses": 21,
+        },
+    ]
+
+
+desika_prabandham_config = {
+    "name": "desika_prabandham",
+    "title": "Desika Prabandham",
+    "category": ScriptureCategoryConstants.LITERATURE,
+    "banner_url": convert_drive_url_to_direct(
+        "https://drive.google.com/file/d/1aRvTo0rOqwWgV_gekntrw1h0B0Hln3Yf/view?usp=drive_link"
+    ),
+    "output_dir": "./output/desika_prabandham",
+    "collection_name": "desika_prabandham",
+    "collection_embedding_fn": "openai",
+    "unit": "verse",
+    "unit_field": "_global_index",
+    "field_mapping": {
+        "text": "lyrics_ta",
+        "unit_index": "relative_verse_number",
+        "chapter_name": "chapter_name",
+        "author": lambda doc: f"ஸ்ரீமந்நிகமாந்த மஹாதேசிகர்",
+        "relative_path": lambda doc: (
+            f"{doc.get('chapter_name')}:{doc.get('relative_verse_number')} || {doc.get('_global_index')} ||"
+        ),
+    },
+    "metadata_fields": [
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter Name",
+            "description": "Name of the Chapter in Tamil",
+            "show_as_filter": True,
+            "component": "dropdown",
+            "lov": lambda: [x["chapter_name"] for x in fn_list_of_chapters()],
+            "is_unique": True,
+        },
+        {
+            "name": "relative_verse_number",
+            "datatype": "int",
+            "label": "Relative Verse Number",
+            "description": "Relative Verse Number",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Absolute Verse Number",
+            "description": "Absolute Verse Number",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "lyrics_ta",
+            "label": "Lyrics in tamil",
+            "datatype": "str",
+            "description": "The original verse in tamil.",
+            "show_as_filter": True,
+        },
+    ],
+    "pdf_path": "./data/desika_prabandham.pdf",
+    "source": "https://www.projectmadurai.org",
+    "language": "tam+eng",
+    "example_labels": [
+        "Desika Prabandham",
+    ],
+    "examples": [
+        "Show some verses from Desika Prabandham",
+    ],
+    "llm_hints": [],
+    "credits": {
+        "art": [
+            {
+                "name": "Sri TNCN",
+                "url": "https://www.linkedin.com/in/narashiman-chakravarthy-54326111/",
+                "role": "Illustrator",
+                "notes": "Original artwork used with permission",
+                "photo_url": "https://drive.google.com/uc?export=download&id=1D4Vo-Mvkp1RdZR6whUkcjvNBIW1yHSUE",
+            }
+        ],
+        "data": [
+            {
+                "name": "Project Madurai",
+                "url": "https://www.projectmadurai.org/",
+                "role": "Data provider",
+            },
+        ],
+        "audio": [],
+        "video": [],
+    },
+}