ChatGPT_Reducer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "mount_file_id": "1zgc3i4hP2g9owwz8aZIlMS671WhJF8cC",
      "authorship_tag": "ABX9TyO5BOQPvE20kOasF8+nzjve",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/milver/Experiments/blob/main/ChatGPT_Reducer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Reduces Size of ChatGPT Files"
      ],
      "metadata": {
        "id": "yD9MjHrsPjKq"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Mount Gdrive"
      ],
      "metadata": {
        "id": "RAzo7dqwSIrx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6wE9eLkNSKsb",
        "outputId": "8cf4291e-bfb0-4619-81cb-7eb7410dbd7a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Import Libraries"
      ],
      "metadata": {
        "id": "2TATWCh8STcx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install ijson\n",
        "!pip install simplejson\n",
        "import json\n",
        "import os\n",
        "import ijson\n",
        "import simplejson as json"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4K3FS6vqPsIq",
        "outputId": "e202b526-186b-498d-f527-2f4a48d59688"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting ijson\n",
            "  Downloading ijson-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (21 kB)\n",
            "Downloading ijson-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n",
            "\u001b[?25l   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/114.5 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K   \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m \u001b[32m112.6/114.5 kB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 kB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: ijson\n",
            "Successfully installed ijson-3.3.0\n",
            "Collecting simplejson\n",
            "  Downloading simplejson-3.19.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.2 kB)\n",
            "Downloading simplejson-3.19.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (137 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m137.9/137.9 kB\u001b[0m \u001b[31m2.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: simplejson\n",
            "Successfully installed simplejson-3.19.3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Give the application permission to mount the drive and create the folders"
      ],
      "metadata": {
        "id": "Vd8XYlaeRqHw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create the Drive folders\n",
        "from google.colab import drive\n",
        "drive.mount(\"/content/drive\", force_remount=True) # This will prompt for authorization.\n",
        "\n",
        "# This will create the folders if they don't exist.\n",
        "folders =  [\"ChatGPT_Conversations/\", \"ChatGPT_Conversations/ProcessedConversations/\", \"ChatGPT_Conversations/ReducedConversations/\"]\n",
        "for folder in folders:\n",
        "  path = \"/content/drive/MyDrive/\" + folder\n",
        "  if not os.path.exists(path): # Create the folder if it does not exist\n",
        "    os.mkdir(path)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GPbFrGvTRwR6",
        "outputId": "ca4e65ef-0dfe-4bd5-cdd2-8eb61074dfbd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Conversation Extractor Functions"
      ],
      "metadata": {
        "id": "rzuL9hUkQOTZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# @title\n",
        "# def process_chat_history(convo_path, output_convo_path):\n",
        "#     \"\"\"\n",
        "#     Extracts conversations with speakers from a chat history file and saves them to a new JSON file.\n",
        "\n",
        "#     Parameters:\n",
        "#         convo_path (str): The path to the chat history file.\n",
        "#         output_convo_path (str): The path to save the extracted conversations.\n",
        "\n",
        "#     Returns:\n",
        "#         None\n",
        "#     \"\"\"\n",
        "#     with open(convo_path, 'r') as file:\n",
        "#         chat_history = json.load(file)\n",
        "\n",
        "#     extracted_conversations_with_speaker = []\n",
        "#     for conversation in chat_history:\n",
        "#     # Start from the current node and traverse backwards\n",
        "#         current_node = conversation.get('current_node')\n",
        "#         while current_node:\n",
        "#             node = conversation.get('mapping', {}).get(current_node)\n",
        "#             if node:\n",
        "#                 message = node.get('message')\n",
        "#                 if message:\n",
        "#                     content = message.get('content', {}).get('parts', [])\n",
        "#                     speaker = message.get('author', {}).get('role')\n",
        "#                     is_user_system_message = message.get('metadata', {}).get('is_user_system_message', False)\n",
        "#                     # Filter out system messages unless they are marked as user system messages\n",
        "#                     if content and speaker and (speaker != \"system\" or is_user_system_message):\n",
        "#                         # Change the speaker's name from \"assistant\" to \"ChatGPT\"\n",
        "#                         if speaker == \"assistant\":\n",
        "#                             speaker = \"ChatGPT\"\n",
        "#                         # Handle the custom user info case for system messages\n",
        "#                         elif speaker == \"system\" and is_user_system_message:\n",
        "#                             speaker = \"Custom user info\"\n",
        "#                         # Include both the content and the speaker in the extracted data\n",
        "#                         for part in content:\n",
        "#                             extracted_conversations_with_speaker.insert(0, {\"speaker\": speaker, \"content\": part})\n",
        "#                 current_node = node.get('parent')\n",
        "\n",
        "#     # Save the new structure to a new JSON file\n",
        "#     with open(output_convo_path, 'w') as new_file:\n",
        "#         json.dump(extracted_conversations_with_speaker, new_file, indent=4)\n",
        "\n",
        "# def process_all_files(input_dir, output_dir):\n",
        "#     \"\"\"\n",
        "#     Process all files in the given input directory and save the processed files\n",
        "#     in the specified output directory.\n",
        "\n",
        "#     Parameters:\n",
        "#     - input_dir (str): The path to the directory containing the input files.\n",
        "#     - output_dir (str): The path to the directory where the processed files will be saved.\n",
        "\n",
        "#     Returns:\n",
        "#     - None\n",
        "#     \"\"\"\n",
        "#     if not os.path.exists(output_dir):\n",
        "#         os.makedirs(output_dir)\n",
        "\n",
        "#     for filename in os.listdir(input_dir):\n",
        "#         if filename.endswith('.json'):\n",
        "#             input_convo_path = os.path.join(input_dir, filename)\n",
        "#             output_convo_path = os.path.join(output_dir, f'processed_{filename}')\n",
        "#             process_chat_history(input_convo_path, output_convo_path)\n",
        "\n",
        "# Usage\n",
        "# split_output_dir = 'split_json'  # Directory where split_json.py saves its output\n",
        "# processed_output_dir = 'processed_json'  # New directory for processed files\n",
        "# process_all_files(split_output_dir, processed_output_dir)"
      ],
      "metadata": {
        "id": "IOPeJ-G_P4fb",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "JSON Splitter"
      ],
      "metadata": {
        "id": "FTDgUYwHQTWT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# @title\n",
        "# def split_json_file(input_file, output_dir, max_size_mb=10):\n",
        "#     \"\"\"\n",
        "#     Split a JSON file into multiple smaller files based on a specified maximum size.\n",
        "\n",
        "#     Args:\n",
        "#         input_file (str): The path to the input JSON file.\n",
        "#         output_dir (str): The directory where the split JSON files will be saved.\n",
        "#         max_size_mb (int, optional): The maximum size in megabytes for each split file. Defaults to 10.\n",
        "\n",
        "#     Raises:\n",
        "#         FileNotFoundError: If the output directory does not exist.\n",
        "\n",
        "#     Returns:\n",
        "#         None\n",
        "#     \"\"\"\n",
        "#     if not os.path.exists(output_dir):\n",
        "#         os.makedirs(output_dir)\n",
        "\n",
        "#     with open(input_file, 'rb') as file:\n",
        "#         file_count = 1\n",
        "#         current_size = 0\n",
        "#         current_data = []\n",
        "\n",
        "#         for item in ijson.items(file, 'item'):\n",
        "#             json_str = json.dumps(item)\n",
        "#             item_size = len(json_str.encode('utf-8'))\n",
        "\n",
        "#             if current_size + item_size > max_size_mb * 1024 * 1024:\n",
        "#                 output_file = os.path.join(output_dir, f'split_{file_count}.json')\n",
        "#                 with open(output_file, 'w') as out_file:\n",
        "#                     json.dump(current_data, out_file)\n",
        "\n",
        "#                 file_count += 1\n",
        "#                 current_size = 0\n",
        "#                 current_data = []\n",
        "\n",
        "#             current_data.append(item)\n",
        "#             current_size += item_size\n",
        "\n",
        "#         if current_data:\n",
        "#             output_file = os.path.join(output_dir, f'split_{file_count}.json')\n",
        "#             with open(output_file, 'w') as out_file:\n",
        "#                 json.dump(current_data, out_file)\n",
        "\n",
        "# Usage\n",
        "# input_file = 'path\\to\\your\\conversations.json'\n",
        "# output_dir = 'split_json'\n",
        "# split_json_file(input_file, output_dir)"
      ],
      "metadata": {
        "id": "I74fGWxzQZyi",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Upload any conversation files you want reduced in the \"ChatGPT_Conversations\" folder in your Google Drive"
      ],
      "metadata": {
        "id": "_7yRrbQwUMED"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Process conversations.json"
      ],
      "metadata": {
        "id": "udBhH3L6REOq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "import os\n",
        "import json\n",
        "from datetime import datetime\n",
        "\n",
        "# Mount Google Drive\n",
        "drive.mount(\"/content/drive\", force_remount=True)  # This will prompt for authorization.\n",
        "\n",
        "# Get the list of JSON files from the ChatGPT_Conversations folder\n",
        "gpt_files = os.listdir(\"/content/drive/MyDrive/ChatGPT_Conversations\")\n",
        "\n",
        "# Function to convert timestamp to readable date\n",
        "def convert_timestamp_to_date(timestamp):\n",
        "    return datetime.utcfromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S') if timestamp else \"Unknown Date\"\n",
        "\n",
        "# Loop through the JSON files and process them\n",
        "for gpt_file in gpt_files:\n",
        "\n",
        "    # Skip the file if it is not a JSON file\n",
        "    if not gpt_file.endswith(\".json\"):\n",
        "        continue\n",
        "\n",
        "    # Load the conversation JSON file\n",
        "    convo_path = \"/content/drive/MyDrive/ChatGPT_Conversations/\" + gpt_file\n",
        "\n",
        "    with open(convo_path, 'r') as file:\n",
        "        chat_history = json.load(file)\n",
        "\n",
        "    # Process the chat history to include title and creation date as parent nodes\n",
        "    extracted_conversations_with_metadata = []\n",
        "\n",
        "    for conversation in chat_history:\n",
        "        # Extract the title and creation date\n",
        "        title = conversation.get(\"title\", \"Untitled Conversation\")\n",
        "        create_time = conversation.get(\"create_time\")\n",
        "        creation_date = convert_timestamp_to_date(create_time)\n",
        "\n",
        "        # Extract the conversation flow\n",
        "        extracted_conversations_with_speaker = []\n",
        "        current_node = conversation.get('current_node')\n",
        "        while current_node:\n",
        "            node = conversation.get('mapping', {}).get(current_node)\n",
        "            if node:\n",
        "                message = node.get('message')\n",
        "                if message:\n",
        "                    content = message.get('content', {}).get('parts', [])\n",
        "                    speaker = message.get('author', {}).get('role')\n",
        "                    is_user_system_message = message.get('metadata', {}).get('is_user_system_message', False)\n",
        "                    # Filter out system messages unless they are marked as user system messages\n",
        "                    if content and speaker and (speaker != \"system\" or is_user_system_message):\n",
        "                        # Change the speaker's name from \"assistant\" to \"ChatGPT\"\n",
        "                        if speaker == \"assistant\":\n",
        "                            speaker = \"ChatGPT\"\n",
        "                        # Handle the custom user info case for system messages\n",
        "                        elif speaker == \"system\" and is_user_system_message:\n",
        "                            speaker = \"Milver Valenzuela\"\n",
        "                        # Include both the content and the speaker in the extracted data\n",
        "                        for part in content:\n",
        "                            extracted_conversations_with_speaker.insert(0, {\"speaker\": speaker, \"content\": part})\n",
        "                current_node = node.get('parent')\n",
        "\n",
        "        # Add the metadata and conversation to the final structure\n",
        "        extracted_conversations_with_metadata.append({\n",
        "            \"title\": title,\n",
        "            \"creation_date\": creation_date,\n",
        "            \"conversation\": extracted_conversations_with_speaker\n",
        "        })\n",
        "\n",
        "    # Save the new structure to a new JSON file\n",
        "    new_convo_path_with_metadata = \"/content/drive/MyDrive/ChatGPT_Conversations/ReducedConversations/reduced_\" + gpt_file\n",
        "    with open(new_convo_path_with_metadata, 'w') as new_file:\n",
        "        json.dump(extracted_conversations_with_metadata, new_file, indent=4)\n",
        "\n",
        "    # Move the original file to the ProcessedConversations folder\n",
        "    processed_path = \"/content/drive/MyDrive/ChatGPT_Conversations/ProcessedConversations/\" + gpt_file\n",
        "    os.rename(convo_path, processed_path)\n",
        "\n",
        "    # Print a message to indicate the progress\n",
        "    print(f\"Processed {gpt_file} and saved the reduced file with metadata in {new_convo_path_with_metadata}\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KKwchlfURJFD",
        "outputId": "a961e49f-d5ce-4262-83ed-0346fe9e0fe9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n",
            "Processed conversations_20241231.json and saved the reduced file with metadata in /content/drive/MyDrive/ChatGPT_Conversations/ReducedConversations/reduced_conversations_20241231.json\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Select and Export conversations into standalone json"
      ],
      "metadata": {
        "id": "bI_c2NOrkoZT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import gradio as gr\n",
        "import json\n",
        "import re\n",
        "import os\n",
        "from datetime import datetime\n",
        "from google.colab import drive\n",
        "\n",
        "# Mount Google Drive\n",
        "drive.mount('/content/drive', force_remount=True)\n",
        "\n",
        "# Helper Functions\n",
        "def load_json(file_path):\n",
        "    \"\"\"Load a JSON file from the specified file path with error handling.\"\"\"\n",
        "    try:\n",
        "        print(\"Starting export process...\")\n",
        "        with open(file_path, 'r', encoding='utf-8') as file:\n",
        "            global chat_data\n",
        "            chat_data = json.load(file)\n",
        "            conversation_list = [\n",
        "                {\n",
        "                    \"title\": conv.get(\"title\", \"Untitled\"),\n",
        "                    \"creation_date\": conv.get(\"creation_date\", \"Unknown Date\")\n",
        "                }\n",
        "                for conv in chat_data\n",
        "            ]\n",
        "            return conversation_list\n",
        "    except FileNotFoundError:\n",
        "        return {\"error\": f\"File not found. Check the path: {file_path}\"}\n",
        "    except json.JSONDecodeError as e:\n",
        "        return {\"error\": f\"Invalid JSON format. {e}\"}\n",
        "    except Exception as e:\n",
        "        return {\"error\": f\"An unexpected error occurred: {e}\"}\n",
        "\n",
        "def search_conversations(query):\n",
        "    \"\"\"Search conversations by title or content with error handling.\"\"\"\n",
        "    try:\n",
        "        if not chat_data:\n",
        "            return {\"error\": \"Please load a JSON file first.\"}\n",
        "\n",
        "        pattern = re.compile(query, re.IGNORECASE)\n",
        "        results = []\n",
        "\n",
        "        for idx, conv in enumerate(chat_data):\n",
        "            if not isinstance(conv, dict):\n",
        "                continue\n",
        "\n",
        "            title = conv.get(\"title\", \"\")\n",
        "            if not isinstance(title, str):\n",
        "                continue\n",
        "\n",
        "            title_match = pattern.search(title)\n",
        "            conversation = conv.get(\"conversation\", [])\n",
        "            snippets = []\n",
        "            if isinstance(conversation, list):\n",
        "                snippets = [\n",
        "                    msg.get(\"content\", \"\")\n",
        "                    for msg in conversation\n",
        "                    if isinstance(msg, dict) and isinstance(msg.get(\"content\", \"\"), str) and pattern.search(msg.get(\"content\", \"\"))\n",
        "                ]\n",
        "\n",
        "            if title_match:\n",
        "                snippets.insert(0, f\"Title Match: {title_match.group(0)}\")\n",
        "\n",
        "            if title_match or snippets:\n",
        "                results.append({\n",
        "                    \"index\": idx,\n",
        "                    \"title\": title,\n",
        "                    \"creation_date\": conv.get(\"creation_date\", \"Unknown Date\"),\n",
        "                    \"snippets\": snippets[:3]\n",
        "                })\n",
        "\n",
        "        return results\n",
        "\n",
        "    except re.error as e:\n",
        "        return {\"error\": f\"Invalid regex pattern: {e}\"}\n",
        "    except Exception as e:\n",
        "        return {\"error\": f\"An unexpected error occurred during the search: {e}\"}\n",
        "\n",
        "def export_selected_conversations(selected_indices, export_folder):\n",
        "    \"\"\"Export selected conversations to JSON files with error handling.\"\"\"\n",
        "    try:\n",
        "        if not chat_data:\n",
        "            return {\"error\": \"Please load a JSON file first.\"}\n",
        "        if not selected_indices:\n",
        "            return {\"error\": \"No conversations selected for export.\"}\n",
        "\n",
        "        if not export_folder.startswith(\"/\"):\n",
        "            export_folder = os.path.join(\"/content/drive/MyDrive\", export_folder)\n",
        "        if not os.path.exists(export_folder):\n",
        "            os.makedirs(export_folder)\n",
        "\n",
        "        exported_files = []\n",
        "        for idx in selected_indices:\n",
        "            if idx < len(chat_data):\n",
        "                conv = chat_data[idx]\n",
        "                title = conv.get(\"title\", \"Untitled\")\n",
        "                creation_date = conv.get(\"creation_date\", \"Unknown_Date\")\n",
        "                sanitized_title = re.sub(r'[\\\\/:*?\"<>|]', '_', title)\n",
        "                file_name = f\"{sanitized_title}_{creation_date.replace(' ', '_')}.json\"\n",
        "                file_path = os.path.join(export_folder, file_name)\n",
        "\n",
        "                with open(file_path, 'w', encoding='utf-8') as file:\n",
        "                    json.dump(conv, file, indent=4)\n",
        "                exported_files.append(file_path)\n",
        "\n",
        "        return {\"message\": f\"Exported {len(exported_files)} conversation(s) to {export_folder}\"}\n",
        "    except Exception as e:\n",
        "        return {\"error\": f\"An unexpected error occurred during export: {e}\"}\n",
        "\n",
        "# Global variable to store chat data\n",
        "chat_data = []\n",
        "\n",
        "# Gradio Interface\n",
        "def handle_export(selected_rows, export_folder):\n",
        "    debug_messages = []\n",
        "    try:\n",
        "        debug_messages.append(\"Export button clicked.\")\n",
        "        debug_messages.append(f\"Selected rows: {selected_rows}\")\n",
        "        debug_messages.append(f\"Export folder: {export_folder}\")\n",
        "\n",
        "        # Ensure selected_rows is a list of rows\n",
        "        if hasattr(selected_rows, \"to_dict\"):\n",
        "            selected_rows = selected_rows.to_dict(orient=\"records\")\n",
        "\n",
        "        if not selected_rows:\n",
        "            debug_messages.append(\"No rows selected for export.\")\n",
        "            return {\"error\": \"No rows selected for export.\", \"debug\": debug_messages}\n",
        "\n",
        "        if not export_folder or not isinstance(export_folder, str):\n",
        "            debug_messages.append(\"Invalid export folder path.\")\n",
        "            return {\"error\": \"Export folder path is not valid.\", \"debug\": debug_messages}\n",
        "\n",
        "        if not chat_data:\n",
        "            debug_messages.append(\"Chat data is not loaded.\")\n",
        "            return {\"error\": \"Please load a JSON file first.\", \"debug\": debug_messages}\n",
        "\n",
        "        debug_messages.append(\"Starting export process...\")\n",
        "\n",
        "        # Extract indices from the first column or 'Index' field\n",
        "        selected_indices = []\n",
        "        for row in selected_rows:\n",
        "            if isinstance(row, dict) and \"Index\" in row:\n",
        "                selected_indices.append(row[\"Index\"])\n",
        "            elif isinstance(row, (list, tuple)) and len(row) > 0:\n",
        "                selected_indices.append(row[0])\n",
        "\n",
        "        debug_messages.append(f\"Selected indices: {selected_indices}\")\n",
        "\n",
        "        # Export the selected conversations\n",
        "        results = export_selected_conversations(selected_indices, export_folder)\n",
        "        debug_messages.append(f\"Export result: {results}\")\n",
        "\n",
        "        if isinstance(results, dict) and \"error\" in results:\n",
        "            debug_messages.append(f\"Export failed with error: {results['error']}\")\n",
        "            return {\"error\": results[\"error\"], \"debug\": debug_messages}\n",
        "\n",
        "        debug_messages.append(\"Export process completed successfully.\")\n",
        "        return {\"message\": results[\"message\"], \"debug\": debug_messages}\n",
        "    except Exception as e:\n",
        "        debug_messages.append(f\"An unexpected error occurred: {e}\")\n",
        "        return {\"error\": f\"An unexpected error occurred: {e}\", \"debug\": debug_messages}\n",
        "\n",
        "\n",
        "# Create Gradio interface\n",
        "with gr.Blocks() as interface:\n",
        "    gr.Markdown(\"# ChatGPT Conversations Manager\")\n",
        "    gr.Markdown(\"Provide the full path to your JSON file in Google Drive, search conversations, and export selected results.\")\n",
        "\n",
        "    file_path_input = gr.Textbox(\n",
        "        label=\"Full Path to JSON File\",\n",
        "        value=\"/content/drive/MyDrive/ChatGPT_Conversations/ReducedConversations/reduced_conversations_20241231.json\"\n",
        "    )\n",
        "    run_app_button = gr.Button(\"Process JSON\")\n",
        "    response_output = gr.JSON(label=\"Response with Debug Messages\")\n",
        "\n",
        "    run_app_button.click(\n",
        "        lambda file_path: load_json(file_path),\n",
        "        inputs=[file_path_input],\n",
        "        outputs=[response_output]\n",
        "    )\n",
        "\n",
        "    search_query_input = gr.Textbox(label=\"Search Query\")\n",
        "    export_folder_input = gr.Textbox(\n",
        "        label=\"Export Folder\",\n",
        "        value=\"/content/drive/MyDrive/ChatGPT_Conversations/IndividualConversations\"\n",
        "    )\n",
        "    search_results_table = gr.Dataframe(\n",
        "        headers=[\"Index\", \"Title\", \"Creation Date\", \"Snippet\"],\n",
        "        value=[],\n",
        "        interactive=True,\n",
        "        datatype=[\"number\", \"str\", \"str\", \"str\"]\n",
        "    )\n",
        "    export_button = gr.Button(\"Export Selected\")\n",
        "\n",
        "    def update_table(query):\n",
        "        search_result = search_conversations(query)\n",
        "        if isinstance(search_result, list):\n",
        "            return [[res['index'], res['title'], res['creation_date'], ' | '.join(res['snippets'])] for res in search_result]\n",
        "        return []\n",
        "\n",
        "    search_query_input.change(\n",
        "        lambda query: update_table(query),\n",
        "        inputs=[search_query_input],\n",
        "        outputs=[search_results_table]\n",
        "    )\n",
        "\n",
        "    export_button.click(\n",
        "        handle_export,\n",
        "        inputs=[search_results_table, export_folder_input],\n",
        "        outputs=[response_output]\n",
        "    )\n",
        "\n",
        "# Launch Gradio app\n",
        "interface.launch(share=True)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 628
        },
        "id": "T3Tz_nOAkuL9",
        "outputId": "6f746ca4-8fde-4fc4-9a96-38846bb701e1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n",
            "Colab notebook detected. To show errors in colab notebook, set debug=True in launch()\n",
            "* Running on public URL: https://3539e812a6e0c1dedd.gradio.live\n",
            "\n",
            "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from the terminal in the working directory to deploy to Hugging Face Spaces (https://huggingface.co/spaces)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<div><iframe src=\"https://3539e812a6e0c1dedd.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": []
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Chunks a conversation into manageable files"
      ],
      "metadata": {
        "id": "25LdYzoVgHpH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# ===================================\n",
        "# 1. Install necessary libraries\n",
        "# ===================================\n",
        "!pip install tiktoken\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Hx3__BgwgWIE",
        "outputId": "bb2693cd-705e-46c2-9735-8a2b11fcf735"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting tiktoken\n",
            "  Downloading tiktoken-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.6 kB)\n",
            "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken) (2024.11.6)\n",
            "Requirement already satisfied: requests>=2.26.0 in /usr/local/lib/python3.10/dist-packages (from tiktoken) (2.32.3)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (3.4.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (2.2.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (2024.12.14)\n",
            "Downloading tiktoken-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
            "\u001b[?25l   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/1.2 MB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K   \u001b[91m━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.1/1.2 MB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K   \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━\u001b[0m \u001b[32m0.9/1.2 MB\u001b[0m \u001b[31m13.1 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m12.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: tiktoken\n",
            "Successfully installed tiktoken-0.8.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# ===================================\n",
        "# 2. Import libraries\n",
        "# ===================================\n",
        "import json\n",
        "import os\n",
        "import zipfile\n",
        "from tiktoken import get_encoding\n",
        "\n",
        "# ===================================\n",
        "# 3. Define helper functions\n",
        "# ===================================\n",
        "def count_tokens(text, encoding_name=\"gpt2\"):\n",
        "    \"\"\"\n",
        "    Counts the number of tokens in a given text using the specified encoding.\n",
        "    By default uses 'gpt2' encoding, which is a good approximation\n",
        "    for GPT-3.5/GPT-4 token usage.\n",
        "    \"\"\"\n",
        "    enc = get_encoding(encoding_name)\n",
        "    return len(enc.encode(text))\n",
        "\n",
        "def build_prompt_response_pairs(conversation):\n",
        "    \"\"\"\n",
        "    Groups a conversation list of dicts into prompt–response pairs.\n",
        "    Example structure of each conversation entry:\n",
        "        {\n",
        "            \"speaker\": \"user\" or \"ChatGPT\",\n",
        "            \"content\": \"...\"\n",
        "        }\n",
        "\n",
        "    Logic:\n",
        "    - For each 'user' message, gather subsequent 'ChatGPT' messages\n",
        "      until the next 'user' appears.\n",
        "    - Each pair is [ user_msg, chatgpt_msg1, chatgpt_msg2, ... ].\n",
        "    - If there's a user without a ChatGPT response, that user remains as a standalone pair\n",
        "      (though typically we want user + response).\n",
        "    \"\"\"\n",
        "    pairs = []\n",
        "    current_user = None\n",
        "    current_chatgpt_responses = []\n",
        "\n",
        "    for msg in conversation:\n",
        "        speaker = msg.get(\"speaker\", \"\").lower()\n",
        "        if speaker == \"user\":\n",
        "            # If there's a pending user + responses, finalize it first\n",
        "            if current_user or current_chatgpt_responses:\n",
        "                # Add the existing user-response pair\n",
        "                # (only if we actually had a user)\n",
        "                if current_user:\n",
        "                    pair = [current_user] + current_chatgpt_responses\n",
        "                    pairs.append(pair)\n",
        "\n",
        "            # Start a new user\n",
        "            current_user = msg\n",
        "            current_chatgpt_responses = []\n",
        "\n",
        "        elif speaker == \"chatgpt\":\n",
        "            # If we have an ongoing user, add this ChatGPT message\n",
        "            if current_user:\n",
        "                current_chatgpt_responses.append(msg)\n",
        "            else:\n",
        "                # No user prompt to attach to; we could ignore or treat it as a leftover\n",
        "                # For safety, you might store these as standalone \"pairs\" if desired.\n",
        "                pass\n",
        "\n",
        "    # After the loop, if there's a leftover user and responses, finalize it\n",
        "    if current_user:\n",
        "        pair = [current_user] + current_chatgpt_responses\n",
        "        pairs.append(pair)\n",
        "\n",
        "    return pairs\n",
        "\n",
        "def pairs_to_text(pairs):\n",
        "    \"\"\"\n",
        "    Given a list of pairs, where each pair is a list of dicts\n",
        "    [ {speaker, content}, {speaker, content}, ... ],\n",
        "    convert everything into a single concatenated string for token counting.\n",
        "    \"\"\"\n",
        "    lines = []\n",
        "    for pair in pairs:\n",
        "        for msg in pair:\n",
        "            speaker = msg.get(\"speaker\", \"\")\n",
        "            content = msg.get(\"content\", \"\")\n",
        "            lines.append(f\"{speaker}: {content}\")\n",
        "    return \"\\n\".join(lines)\n",
        "\n",
        "# ===================================\n",
        "# 4. Main chunking + zipping logic\n",
        "# ===================================\n",
        "def chunk_json_file_into_pairs(\n",
        "    input_json_path,\n",
        "    output_prefix=\"my_conversation_chunk\",\n",
        "    max_tokens=7000,          # ~7k is safer for GPT-4 8k context\n",
        "    encoding_name=\"gpt2\"\n",
        "):\n",
        "    \"\"\"\n",
        "    - Loads a JSON file with:\n",
        "        {\n",
        "          \"title\": ...,\n",
        "          \"creation_date\": ...,\n",
        "          \"conversation\": [\n",
        "             { \"speaker\": ..., \"content\": ... },\n",
        "             ...\n",
        "          ]\n",
        "        }\n",
        "    - Groups the conversation into user->ChatGPT prompt–response pairs.\n",
        "    - Chunks those pairs so that each output JSON stays under `max_tokens` tokens.\n",
        "    - Each chunk is saved to disk and then all chunk files are zipped into:\n",
        "        \"chunked_<original-filename>.zip\".\n",
        "    \"\"\"\n",
        "    # 1. Load the input JSON\n",
        "    with open(input_json_path, \"r\", encoding=\"utf-8\") as f:\n",
        "        data = json.load(f)\n",
        "\n",
        "    title = data.get(\"title\", \"\")\n",
        "    creation_date = data.get(\"creation_date\", \"\")\n",
        "    conversation = data.get(\"conversation\", [])\n",
        "\n",
        "    # 2. Build prompt–response pairs\n",
        "    all_pairs = build_prompt_response_pairs(conversation)\n",
        "\n",
        "    # We'll accumulate pairs into a chunk until it exceeds max_tokens\n",
        "    chunk_filenames = []\n",
        "    chunk_counter = 1\n",
        "\n",
        "    current_chunk = []\n",
        "    current_text = \"\"\n",
        "\n",
        "    for pair in all_pairs:\n",
        "        # Turn this pair into text for token count\n",
        "        pair_text = pairs_to_text([pair])\n",
        "        test_text = current_text + \"\\n\" + pair_text\n",
        "        test_token_count = count_tokens(test_text, encoding_name=encoding_name)\n",
        "\n",
        "        if test_token_count <= max_tokens:\n",
        "            # Add this pair to the current chunk\n",
        "            current_chunk.append(pair)\n",
        "            current_text = test_text\n",
        "        else:\n",
        "            # Save the current chunk, then start a new one\n",
        "            if current_chunk:\n",
        "                # Build chunk data in the same schema as the original,\n",
        "                # but with conversation replaced by the chunk pairs flattened\n",
        "                chunk_data = {\n",
        "                    \"title\": title,\n",
        "                    \"creation_date\": creation_date,\n",
        "                    \"conversation\": [msg for pair_ in current_chunk for msg in pair_]\n",
        "                }\n",
        "\n",
        "                chunk_filename = f\"{output_prefix}_{chunk_counter:03d}.json\"\n",
        "                with open(chunk_filename, \"w\", encoding=\"utf-8\") as out_f:\n",
        "                    json.dump(chunk_data, out_f, ensure_ascii=False, indent=4)\n",
        "\n",
        "                print(f\"Saved chunk #{chunk_counter}: {chunk_filename} \"\n",
        "                      f\"(~{count_tokens(pairs_to_text(current_chunk), encoding_name)} tokens)\")\n",
        "                chunk_filenames.append(chunk_filename)\n",
        "                chunk_counter += 1\n",
        "\n",
        "            # Start a fresh chunk with the new pair\n",
        "            current_chunk = [pair]\n",
        "            current_text = pair_text\n",
        "\n",
        "    # Save any leftover chunk\n",
        "    if current_chunk:\n",
        "        chunk_data = {\n",
        "            \"title\": title,\n",
        "            \"creation_date\": creation_date,\n",
        "            \"conversation\": [msg for pair_ in current_chunk for msg in pair_]\n",
        "        }\n",
        "        chunk_filename = f\"{output_prefix}_{chunk_counter:03d}.json\"\n",
        "        with open(chunk_filename, \"w\", encoding=\"utf-8\") as out_f:\n",
        "            json.dump(chunk_data, out_f, ensure_ascii=False, indent=4)\n",
        "\n",
        "        print(f\"Saved final chunk #{chunk_counter}: {chunk_filename} \"\n",
        "              f\"(~{count_tokens(pairs_to_text(current_chunk), encoding_name)} tokens)\")\n",
        "        chunk_filenames.append(chunk_filename)\n",
        "\n",
        "    # 3. Zip all the chunks\n",
        "    base_name = os.path.splitext(os.path.basename(input_json_path))[0]\n",
        "    zip_filename = f\"chunked_{base_name}.zip\"\n",
        "    with zipfile.ZipFile(zip_filename, 'w') as zf:\n",
        "        for cf in chunk_filenames:\n",
        "            zf.write(cf)\n",
        "\n",
        "    print(f\"\\nAll chunks have been zipped into: {zip_filename}\")\n",
        "    print(\"Chunk files included:\")\n",
        "    for cf in chunk_filenames:\n",
        "        print(f\"  - {cf}\")\n",
        "\n",
        "# ===================================\n",
        "# 5. Example usage in Colab\n",
        "# ===================================\n",
        "\n",
        "# Replace 'transcripts.json' with the actual file you have in your Colab environment.\n",
        "input_file_path = \"/content/drive/MyDrive/ChatGPT_Conversations/IndividualConversations/LinkedIn Networking and Recruiting_2024-08-15_17:13:45.json\"\n",
        "\n",
        "# This function will:\n",
        "#  - read the JSON\n",
        "#  - group conversation into pairs\n",
        "#  - chunk them up to ~7000 tokens each (safe for GPT-4 8K limit)\n",
        "#  - save each chunk as a JSON file\n",
        "#  - zip them all into chunked_transcripts.zip\n",
        "chunk_json_file_into_pairs(\n",
        "    input_json_path=input_file_path,\n",
        "    output_prefix=\"my_conversation_chunk\",\n",
        "    max_tokens=7000,   # ~7K tokens for safety\n",
        "    encoding_name=\"gpt2\"\n",
        ")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y4uDyVZ2gQc7",
        "outputId": "c997d4fb-40bb-4a88-a4b7-ba85e75179a1"
      },
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saved chunk #1: my_conversation_chunk_001.json (~6586 tokens)\n",
            "Saved chunk #2: my_conversation_chunk_002.json (~6388 tokens)\n",
            "Saved chunk #3: my_conversation_chunk_003.json (~4240 tokens)\n",
            "Saved chunk #4: my_conversation_chunk_004.json (~5337 tokens)\n",
            "Saved chunk #5: my_conversation_chunk_005.json (~6188 tokens)\n",
            "Saved chunk #6: my_conversation_chunk_006.json (~6757 tokens)\n",
            "Saved final chunk #7: my_conversation_chunk_007.json (~3444 tokens)\n",
            "\n",
            "All chunks have been zipped into: chunked_LinkedIn Networking and Recruiting_2024-08-15_17:13:45.zip\n",
            "Chunk files included:\n",
            "  - my_conversation_chunk_001.json\n",
            "  - my_conversation_chunk_002.json\n",
            "  - my_conversation_chunk_003.json\n",
            "  - my_conversation_chunk_004.json\n",
            "  - my_conversation_chunk_005.json\n",
            "  - my_conversation_chunk_006.json\n",
            "  - my_conversation_chunk_007.json\n"
          ]
        }
      ]
    }
  ]
}