AICoverSong.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/reshalfahsi/AI-Cover-Song/blob/master/AICoverSong.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## AI Cover Song\n",
        "\n",
        "Just Run all (Ctrl + F9)"
      ],
      "metadata": {
        "id": "vLbqtahwoHsW"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "1sEsl0KoNJMN"
      },
      "outputs": [],
      "source": [
        "#@title Prerequisite\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "\n",
        "!pip install pydub\n",
        "!pip install yt_dlp\n",
        "!pip install ffmpeg\n",
        "!python3 -m pip install -U demucs\n",
        "!python -m pip install -U pip wheel\n",
        "%pip install -U ipython\n",
        "%pip install -U so-vits-svc-fork\n",
        "\n",
        "\n",
        "!mkdir -p drive/MyDrive/so-vits-svc-fork\n",
        "!rm -rf drive/MyDrive/so-vits-svc-fork\n",
        "!mkdir drive/MyDrive/so-vits-svc-fork\n",
        "\n",
        "\n",
        "!mkdir -p youtubeaudio\n",
        "!mkdir -p dataset_raw\n",
        "!mkdir -p dataset"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Download Youtube WAV\n",
        "from __future__ import unicode_literals\n",
        "import yt_dlp\n",
        "import ffmpeg\n",
        "import sys\n",
        "\n",
        "\n",
        "def download_from_url(url, output_dir):\n",
        "    ydl_opts = {\n",
        "        'format': 'bestaudio/best',\n",
        "        'postprocessors': [{\n",
        "            'key': 'FFmpegExtractAudio',\n",
        "            'preferredcodec': 'wav',\n",
        "        }],\n",
        "        \"outtmpl\": output_dir,\n",
        "    }\n",
        "    with yt_dlp.YoutubeDL(ydl_opts) as ydl:\n",
        "        ydl.download([url])"
      ],
      "metadata": {
        "cellView": "form",
        "id": "cglLWrh7Nbnp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Separate Vocal and Instrument/Noise using Demucs\n",
        "import subprocess\n",
        "\n",
        "\n",
        "def separate_vocal_and_instrument(audio_input):\n",
        "    command = f\"demucs --two-stems=vocals {audio_input}\"\n",
        "    result = subprocess.run(command.split(), stdout=subprocess.PIPE)\n",
        "    print(result.stdout.decode())"
      ],
      "metadata": {
        "cellView": "form",
        "id": "eZhNNlPZOLIM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Song\n",
        "\n",
        "SONG_URL = \"https://www.youtube.com/watch?v=cQGfLDnmWS8&pp=ygULYXNtYWxpYnJhc2k%3D\" # @param {type:\"string\"}\n",
        "SONG_PATH = 'youtubeaudio'\n",
        "download_from_url(\n",
        "    url=SONG_URL,\n",
        "    output_dir=f\"{SONG_PATH}/audio\",\n",
        "    )\n",
        "%cd -q $SONG_PATH\n",
        "separate_vocal_and_instrument(audio_input=f\"/content/{SONG_PATH}/audio.wav\")\n",
        "%cd /content"
      ],
      "metadata": {
        "cellView": "form",
        "id": "e0IVziDeOaaa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Split The Audio Dataset (Speaker) into Smaller Duration Before Training\n",
        "\n",
        "SPEAKER_NAME = \"ItsukiNakano\" # @param {type:\"string\"}\n",
        "DATASET_URL = \"https://www.youtube.com/watch?v=oR7hJx7h0WQ\" # @param{type:\"string\"}\n",
        "DATASET_PATH = 'dataset'\n",
        "download_from_url(\n",
        "    url=DATASET_URL,\n",
        "    output_dir=f\"{DATASET_PATH}/audio\",\n",
        "    )\n",
        "%cd -q $DATASET_PATH\n",
        "separate_vocal_and_instrument(audio_input=f\"/content/{DATASET_PATH}/audio.wav\")\n",
        "%cd /content\n",
        "\n",
        "DATASET_RAW = f'dataset_raw/{SPEAKER_NAME}'\n",
        "\n",
        "%mkdir -p $DATASET_RAW\n",
        "\n",
        "\n",
        "from scipy.io import wavfile\n",
        "import os\n",
        "import numpy as np\n",
        "import argparse\n",
        "from tqdm import tqdm\n",
        "import json\n",
        "\n",
        "from datetime import datetime, timedelta\n",
        "\n",
        "# Utility functions\n",
        "\n",
        "\n",
        "def GetTime(video_seconds):\n",
        "    if video_seconds < 0:\n",
        "        return 00\n",
        "\n",
        "    else:\n",
        "        sec = timedelta(seconds=float(video_seconds))\n",
        "        d = datetime(1, 1, 1) + sec\n",
        "\n",
        "        instant = (\n",
        "            str(d.hour).zfill(2)\n",
        "            + \":\"\n",
        "            + str(d.minute).zfill(2)\n",
        "            + \":\"\n",
        "            + str(d.second).zfill(2)\n",
        "            + str(\".001\")\n",
        "        )\n",
        "\n",
        "        return instant\n",
        "\n",
        "\n",
        "def GetTotalTime(video_seconds):\n",
        "    sec = timedelta(seconds=float(video_seconds))\n",
        "    d = datetime(1, 1, 1) + sec\n",
        "    delta = str(d.hour) + \":\" + str(d.minute) + \":\" + str(d.second)\n",
        "\n",
        "    return delta\n",
        "\n",
        "\n",
        "def windows(signal, window_size, step_size):\n",
        "    if type(window_size) is not int:\n",
        "        raise AttributeError(\"Window size must be an integer.\")\n",
        "    if type(step_size) is not int:\n",
        "        raise AttributeError(\"Step size must be an integer.\")\n",
        "    for i_start in range(0, len(signal), step_size):\n",
        "        i_end = i_start + window_size\n",
        "        if i_end >= len(signal):\n",
        "            break\n",
        "        yield signal[i_start:i_end]\n",
        "\n",
        "\n",
        "def energy(samples):\n",
        "    return np.sum(np.power(samples, 2.0)) / float(len(samples))\n",
        "\n",
        "\n",
        "def rising_edges(binary_signal):\n",
        "    previous_value = 0\n",
        "    index = 0\n",
        "    for x in binary_signal:\n",
        "        if x and not previous_value:\n",
        "            yield index\n",
        "        previous_value = x\n",
        "        index += 1\n",
        "\n",
        "\n",
        "\"\"\"\n",
        "Last Acceptable Values\n",
        "\n",
        "min_silence_length = 0.3\n",
        "silence_threshold = 1e-3\n",
        "step_duration = 0.03/10\n",
        "\n",
        "\"\"\"\n",
        "# Change the arguments and the input file here\n",
        "input_file = \"/content/dataset/separated/htdemucs/audio/vocals.wav\"\n",
        "output_dir = f\"/content/dataset_raw/{SPEAKER_NAME}\"\n",
        "min_silence_length = 0.6  # The minimum length of silence at which a split may occur [seconds]. Defaults to 3 seconds.\n",
        "silence_threshold = 1e-4  # The energy level (between 0.0 and 1.0) below which the signal is regarded as silent.\n",
        "step_duration = (\n",
        "    0.03 / 10\n",
        ")  # The amount of time to step forward in the input file after calculating energy. Smaller value = slower, but more accurate silence detection. Larger value = faster, but might miss some split opportunities. Defaults to (min-silence-length / 10.).\n",
        "\n",
        "\n",
        "input_filename = input_file\n",
        "window_duration = min_silence_length\n",
        "if step_duration is None:\n",
        "    step_duration = window_duration / 10.0\n",
        "else:\n",
        "    step_duration = step_duration\n",
        "\n",
        "output_filename_prefix = os.path.splitext(os.path.basename(input_filename))[0]\n",
        "dry_run = False\n",
        "\n",
        "print(\n",
        "    \"Splitting {} where energy is below {}% for longer than {}s.\".format(\n",
        "        input_filename, silence_threshold * 100.0, window_duration\n",
        "    )\n",
        ")\n",
        "\n",
        "# Read and split the file\n",
        "\n",
        "sample_rate, samples = wavfile.read(filename=input_filename, mmap=True)\n",
        "\n",
        "max_amplitude = np.iinfo(samples.dtype).max\n",
        "print(max_amplitude)\n",
        "\n",
        "max_energy = energy([max_amplitude])\n",
        "print(max_energy)\n",
        "\n",
        "window_size = int(window_duration * sample_rate)\n",
        "step_size = int(step_duration * sample_rate)\n",
        "\n",
        "signal_windows = windows(signal=samples, window_size=window_size, step_size=step_size)\n",
        "\n",
        "window_energy = (\n",
        "    energy(w) / max_energy\n",
        "    for w in tqdm(signal_windows, total=int(len(samples) / float(step_size)))\n",
        ")\n",
        "\n",
        "window_silence = (e > silence_threshold for e in window_energy)\n",
        "\n",
        "cut_times = (r * step_duration for r in rising_edges(window_silence))\n",
        "\n",
        "# This is the step that takes long, since we force the generators to run.\n",
        "print(\"Finding silences...\")\n",
        "cut_samples = [int(t * sample_rate) for t in cut_times]\n",
        "cut_samples.append(-1)\n",
        "\n",
        "cut_ranges = [\n",
        "    (i, cut_samples[i], cut_samples[i + 1]) for i in range(len(cut_samples) - 1)\n",
        "]\n",
        "\n",
        "video_sub = {\n",
        "    str(i): [\n",
        "        str(GetTime(((cut_samples[i]) / sample_rate))),\n",
        "        str(GetTime(((cut_samples[i + 1]) / sample_rate))),\n",
        "    ]\n",
        "    for i in range(len(cut_samples) - 1)\n",
        "}\n",
        "\n",
        "for i, start, stop in tqdm(cut_ranges):\n",
        "    output_file_path = \"{}_{:03d}.wav\".format(\n",
        "        os.path.join(output_dir, output_filename_prefix), i\n",
        "    )\n",
        "    if not dry_run:\n",
        "        print(\"Writing file {}\".format(output_file_path))\n",
        "        wavfile.write(\n",
        "            filename=output_file_path, rate=sample_rate, data=samples[start:stop]\n",
        "        )\n",
        "    else:\n",
        "        print(\"Not writing file {}\".format(output_file_path))\n",
        "\n",
        "with open(output_dir + \"\\\\\" + output_filename_prefix + \".json\", \"w\") as output:\n",
        "    json.dump(video_sub, output)\n"
      ],
      "metadata": {
        "cellView": "form",
        "id": "DFWtM9DJRnGq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Automatic preprocessing\n",
        "!svc pre-resample\n",
        "!svc pre-config"
      ],
      "metadata": {
        "cellView": "form",
        "id": "ILksYBDMX8dh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Copy configs file\n",
        "!cp configs/44k/config.json drive/MyDrive/so-vits-svc-fork"
      ],
      "metadata": {
        "cellView": "form",
        "id": "wM5pKgOlZeDc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "F0_METHOD = \"dio\" #@param [\"crepe\", \"crepe-tiny\", \"parselmouth\", \"dio\", \"harvest\"]\n",
        "!svc pre-hubert -fm {F0_METHOD}"
      ],
      "metadata": {
        "cellView": "form",
        "id": "eGWlF7CkZ5pV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Train\n",
        "%load_ext tensorboard\n",
        "%tensorboard --logdir drive/MyDrive/so-vits-svc-fork/logs/44k\n",
        "!svc train --model-path drive/MyDrive/so-vits-svc-fork/logs/44k"
      ],
      "metadata": {
        "cellView": "form",
        "id": "rrgqexOiZ9Y1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Inference\n",
        "# remove \".wav\" on AUDIO\n",
        "from IPython.display import Audio\n",
        "\n",
        "AUDIO = \"/content/youtubeaudio/separated/htdemucs/audio/vocals\"\n",
        "MODEL = \"/content/drive/MyDrive/so-vits-svc-fork/logs/44k/G_10000.pth\" #@param {type:\"string\"}\n",
        "CONFIG = \"/content/drive/MyDrive/so-vits-svc-fork/logs/44k/config.json\"\n",
        "#@markdown Change According to Your Voice Tone. 12 = 1 Octave | -12 = -1 Octave\n",
        "PITCH = 0 #@param {type:\"integer\"}\n",
        "\n",
        "!svc infer {AUDIO}.wav -c {CONFIG} -m {MODEL} -na -t {PITCH}"
      ],
      "metadata": {
        "cellView": "form",
        "id": "0RhQu93waH5c"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Combine Vocal and Instrument (Song Cover)\n",
        "from pydub import AudioSegment\n",
        "\n",
        "VOCAL = \"/content/youtubeaudio/separated/htdemucs/audio/vocals.out.wav\"\n",
        "INSTRUMENT = \"/content/youtubeaudio/separated/htdemucs/audio/no_vocals.wav\"\n",
        "\n",
        "sound1 = AudioSegment.from_file(VOCAL)\n",
        "sound2 = AudioSegment.from_file(INSTRUMENT)\n",
        "\n",
        "combined = sound1.overlay(sound2)\n",
        "\n",
        "combined.export(\"/content/FinalCover.wav\", format='wav')\n",
        "print(\"Saved to /content/FinalCover.wav\")"
      ],
      "metadata": {
        "cellView": "form",
        "id": "fkt-NKzPf4_c"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}