GITScrapper202403PO.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GIT Metadata Scrapper\n",
    "# Code Generated with help from ChatGPT\n",
    "# Created on 2024/03/13\n",
    "# Prompted by Priyanka"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Example: List repositories of a user\n",
    "username = 'priya-gitTest'\n",
    "url = f'https://api.github.com/users/{username}/repos'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replace 'your_token_here' with your actual token\n",
    "token = 'xxx'\n",
    "# Replace 'owner_name' with the owner's GitHub username\n",
    "owner = 'priya-gitTest'\n",
    "# Replace 'repo_name' with the name of the repository\n",
    "repo_name = 'jepa'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_repo_info(owner, repo_name, token):\n",
    "    url = f'https://api.github.com/repos/{owner}/{repo_name}'\n",
    "    headers = {\n",
    "        'Authorization': f'token {token}',\n",
    "        'Accept': 'application/vnd.github.v3+json'\n",
    "    }\n",
    "\n",
    "    response = requests.get(url, headers=headers)\n",
    "\n",
    "    if response.status_code == 200:\n",
    "        repo_info = response.json()\n",
    "        return repo_info\n",
    "    else:\n",
    "        print(f\"Failed to retrieve repository information: {response.status_code}\")\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Repository Information:\n",
      "Name: jepa\n",
      "Owner: priya-gitTest\n",
      "Description: PyTorch code and models for V-JEPA self-supervised learning from video.\n",
      "URL: https://github.com/priya-gitTest/jepa\n",
      "Language: None\n",
      "Forks Count: 0\n",
      "Stars Count: 0\n",
      "Watchers Count: 0\n"
     ]
    }
   ],
   "source": [
    "repo_info = get_repo_info(owner, repo_name, token)\n",
    "if repo_info:\n",
    "    print(\"Repository Information:\")\n",
    "    print(f\"Name: {repo_info['name']}\")\n",
    "    print(f\"Owner: {repo_info['owner']['login']}\")\n",
    "    print(f\"Description: {repo_info['description']}\")\n",
    "    print(f\"URL: {repo_info['html_url']}\")\n",
    "    print(f\"Language: {repo_info['language']}\")\n",
    "    print(f\"Forks Count: {repo_info['forks_count']}\")\n",
    "    print(f\"Stars Count: {repo_info['stargazers_count']}\")\n",
    "    print(f\"Watchers Count: {repo_info['watchers_count']}\")\n",
    "else:\n",
    "    print(\"Failed to retrieve repository information.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_repo_languages(owner, repo_name, token):\n",
    "    url = f'https://api.github.com/repos/{owner}/{repo_name}/languages'\n",
    "    headers = {\n",
    "        'Authorization': f'token {token}',\n",
    "        'Accept': 'application/vnd.github.v3+json'\n",
    "    }\n",
    "\n",
    "    response = requests.get(url, headers=headers)\n",
    "\n",
    "    if response.status_code == 200:\n",
    "        languages = response.json()\n",
    "        return languages\n",
    "    else:\n",
    "        print(f\"Failed to retrieve repository languages: {response.status_code}\")\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Languages Used in the Repository:\n",
      "Python: 245282 bytes\n"
     ]
    }
   ],
   "source": [
    "repo_languages = get_repo_languages(owner, repo_name, token)\n",
    "if repo_languages:\n",
    "    print(\"Languages Used in the Repository:\")\n",
    "    for language, bytes_of_code in repo_languages.items():\n",
    "        print(f\"{language}: {bytes_of_code} bytes\")\n",
    "else:\n",
    "    print(\"Failed to retrieve repository languages.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_repo_archive_url(owner, repo_name, archive_format, token):\n",
    "    url = f'https://api.github.com/repos/{owner}/{repo_name}/tarball/main'\n",
    "    headers = {\n",
    "        'Authorization': f'token {token}',\n",
    "        'Accept': 'application/vnd.github.v3+json'\n",
    "    }\n",
    "\n",
    "    response = requests.get(url, headers=headers)\n",
    "\n",
    "    if response.status_code == 200:\n",
    "        archive_url = response.url\n",
    "        return archive_url\n",
    "    else:\n",
    "        print(f\"Failed to retrieve repository archive URL: {response.status_code}\")\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Archive URL for priya-gitTest/jepa:\n",
      "https://codeload.github.com/priya-gitTest/jepa/legacy.tar.gz/refs/heads/main\n"
     ]
    }
   ],
   "source": [
    "# Specify the desired archive format ('tarball' or 'zipball')\n",
    "archive_format = 'tarball'\n",
    "\n",
    "archive_url = get_repo_archive_url(owner, repo_name, archive_format, token)\n",
    "if archive_url:\n",
    "    print(f\"Archive URL for {owner}/{repo_name}:\")\n",
    "    print(archive_url)\n",
    "else:\n",
    "    print(\"Failed to retrieve repository archive URL.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_repo_license(owner, repo_name, token):\n",
    "    url = f'https://api.github.com/repos/{owner}/{repo_name}/license'\n",
    "    headers = {\n",
    "        'Authorization': f'token {token}',\n",
    "        'Accept': 'application/vnd.github.v3+json'\n",
    "    }\n",
    "\n",
    "    response = requests.get(url, headers=headers)\n",
    "\n",
    "    if response.status_code == 200:\n",
    "        license_info = response.json()\n",
    "        return license_info\n",
    "    elif response.status_code == 404:\n",
    "        print(\"Repository does not have a license.\")\n",
    "        return None\n",
    "    else:\n",
    "        print(f\"Failed to retrieve repository license: {response.status_code}\")\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "License Information:\n",
      "Name: LICENSE\n",
      "SPDX ID: N/A\n",
      "URL: https://github.com/priya-gitTest/jepa/blob/main/LICENSE\n"
     ]
    }
   ],
   "source": [
    "license_info = get_repo_license(owner, repo_name, token)\n",
    "if license_info:\n",
    "    print(\"License Information:\")\n",
    "    print(f\"Name: {license_info.get('name', 'N/A')}\")\n",
    "    print(f\"SPDX ID: {license_info.get('spdx_id', 'N/A')}\")\n",
    "    print(f\"URL: {license_info.get('html_url', 'N/A')}\")\n",
    "else:\n",
    "    print(\"Failed to retrieve repository license.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"codeRepository\": \"https://api.github.com/repos/priya-gitTest/jepa\",\n",
      "    \"programmingLanguage\": \"Python\",\n",
      "    \"downloadUrl\": \"https://codeload.github.com/priya-gitTest/jepa/legacy.tar.gz/refs/heads/main\",\n",
      "    \"author\": \"priya-gitTest\",\n",
      "    \"license\": \"https://github.com/priya-gitTest/jepa/blob/main/LICENSE\"\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "# Create a dictionary representing the author information\n",
    "url = f'https://api.github.com/repos/{owner}/{repo_name}'\n",
    "author_info = {\n",
    "    \"codeRepository\":url,\n",
    "    \"programmingLanguage\": language,\n",
    "    \"downloadUrl\": archive_url,\n",
    "    \"author\": owner,\n",
    "    \"license\":license_info.get('html_url', 'N/A')\n",
    "}\n",
    "\n",
    "# Serialize the dictionary to JSON\n",
    "json_ld = json.dumps(author_info, indent=4)\n",
    "\n",
    "# Print the JSON-LD\n",
    "print(json_ld)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "### TODO Make a JSON output like this : https://github.com/ropensci/codemetar/blob/main/codemeta.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The watermark extension is already loaded. To reload it, use:\n",
      "  %reload_ext watermark\n"
     ]
    }
   ],
   "source": [
    "#WaterMark for Notebook.\n",
    "#Ref: https://github.com/jupyter-guide/jupyter-guide/blob/master/notebooks/Rule5_Watermark.ipynb \n",
    "\n",
    "#%watermark\n",
    "\n",
    "%load_ext watermark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python implementation: CPython\n",
      "Python version       : 3.10.6\n",
      "IPython version      : 8.22.2\n",
      "\n",
      "Compiler    : MSC v.1932 64 bit (AMD64)\n",
      "OS          : Windows\n",
      "Release     : 10\n",
      "Machine     : AMD64\n",
      "Processor   : Intel64 Family 6 Model 142 Stepping 9, GenuineIntel\n",
      "CPU cores   : 4\n",
      "Architecture: 64bit\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#%watermark\n",
    "%watermark -v -m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "json    : 2.0.9\n",
      "requests: 2.31.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#%watermark -p pandas,numpy,matplotlib\n",
    "#%watermark -p requests\n",
    "#%watermark -p json\n",
    "%watermark -iv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}