mirror of
https://github.com/ION606/youtube-music-meta-extract.git
synced 2026-05-14 22:06:56 +00:00
initial code commit
This commit is contained in:
Binary file not shown.
|
After Width: | Height: | Size: 20 KiB |
@@ -0,0 +1,263 @@
|
||||
import express from 'express';
|
||||
import { google } from 'googleapis';
|
||||
import open from 'open';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { tokenManager } from './tokenManager.js';
|
||||
|
||||
(await import('dotenv')).config({
|
||||
path: './Downloader/secret/config.env',
|
||||
debug: true
|
||||
});
|
||||
|
||||
|
||||
const app = express();
|
||||
const port = 3000;
|
||||
|
||||
|
||||
const { CLIENT_ID, CLIENT_SECRET, REDIRECT_URI } = process.env,
|
||||
manager = new tokenManager({ clientId: CLIENT_ID, clientSecret: CLIENT_SECRET, redirectUri: REDIRECT_URI, tokenPath: 'Downloader/secret/token.json' });
|
||||
|
||||
const oauth2Client = manager.getAuthClient();
|
||||
|
||||
|
||||
// scope to read playlist items/liked videos
|
||||
const SCOPES = ['https://www.googleapis.com/auth/youtube.readonly'];
|
||||
|
||||
let downloadStatus = 'idle'; // can be: 'idle', 'in-progress', 'completed', 'error'
|
||||
|
||||
|
||||
//#region oauth flow
|
||||
|
||||
app.get('/auth', async (_req, res) => {
|
||||
const t = manager.loadToken();
|
||||
if (t) return res.redirect('/choose-playlist');
|
||||
|
||||
// generate auth url
|
||||
const authUrl = oauth2Client.generateAuthUrl({
|
||||
access_type: 'offline',
|
||||
scope: SCOPES
|
||||
});
|
||||
|
||||
// automatically open the url in the default browser
|
||||
const c = await open(authUrl).catch((err) => {
|
||||
console.error('error opening browser:', err);
|
||||
return res.status(500).send('failed to open browser for oauth.');
|
||||
});
|
||||
|
||||
c.on('close', () => res.redirect('/choose-playlist'))
|
||||
});
|
||||
|
||||
|
||||
|
||||
app.get('/oauth2callback', async (req, res) => {
|
||||
try {
|
||||
const code = req.query.code;
|
||||
const { tokens } = await oauth2Client.getToken(code);
|
||||
oauth2Client.setCredentials(tokens);
|
||||
|
||||
manager.saveToken(tokens);
|
||||
|
||||
// close the window
|
||||
res.sendStatus(200);
|
||||
} catch (err) {
|
||||
console.error('error retrieving token:', err);
|
||||
res.status(500).send('error retrieving token.');
|
||||
}
|
||||
});
|
||||
|
||||
//#endregion
|
||||
|
||||
|
||||
//#region youtube stuffs
|
||||
|
||||
async function getAllPlaylists(auth) {
|
||||
const youtube = google.youtube('v3');
|
||||
let playlists = [];
|
||||
let nextPageToken = null;
|
||||
|
||||
do {
|
||||
const response = await youtube.playlists.list({
|
||||
auth,
|
||||
part: 'snippet',
|
||||
mine: true,
|
||||
maxResults: 50,
|
||||
pageToken: nextPageToken
|
||||
});
|
||||
|
||||
if (response.data.items) {
|
||||
playlists = playlists.concat(response.data.items);
|
||||
}
|
||||
|
||||
nextPageToken = response.data.nextPageToken;
|
||||
} while (nextPageToken);
|
||||
|
||||
return playlists;
|
||||
}
|
||||
|
||||
async function getPlaylistItems(playlistId, auth) {
|
||||
const youtube = google.youtube('v3');
|
||||
let items = [];
|
||||
let nextPageToken = null;
|
||||
|
||||
do {
|
||||
const response = await youtube.playlistItems.list({
|
||||
auth,
|
||||
part: 'snippet,contentDetails',
|
||||
playlistId,
|
||||
maxResults: 50,
|
||||
pageToken: nextPageToken
|
||||
});
|
||||
|
||||
if (response.data.items) {
|
||||
items = items.concat(response.data.items);
|
||||
}
|
||||
|
||||
nextPageToken = response.data.nextPageToken;
|
||||
} while (nextPageToken);
|
||||
|
||||
return items.map(o => `https://music.youtube.com/watch?v=${o.id}`);
|
||||
}
|
||||
|
||||
//#endregion
|
||||
|
||||
|
||||
//#region routes
|
||||
|
||||
app.get('/choose-playlist', async (_req, res) => {
|
||||
try {
|
||||
if (!oauth2Client.credentials || !oauth2Client.credentials.access_token) {
|
||||
const t = manager.loadToken();
|
||||
if (!t) return res.redirect('/auth');
|
||||
}
|
||||
|
||||
const playlists = await getAllPlaylists(oauth2Client);
|
||||
|
||||
let html = `
|
||||
<html>
|
||||
<head>
|
||||
<title>choose playlist</title>
|
||||
<style>
|
||||
body { font-family: sans-serif; }
|
||||
#container { margin: 20px; }
|
||||
select, button { margin-top: 10px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="container">
|
||||
<h1>choose a playlist to download</h1>
|
||||
<select id="playlistSelect">
|
||||
${playlists
|
||||
.map(
|
||||
(pl) =>
|
||||
`<option value="${pl.id}">${pl.snippet.title}</option>`
|
||||
)
|
||||
.join('')
|
||||
}
|
||||
</select>
|
||||
<br/>
|
||||
<button id="downloadBtn">download playlist</button>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// when the button is clicked, we'll navigate to /download-playlist?playlistId=...
|
||||
const downloadBtn = document.querySelector('#downloadBtn');
|
||||
const select = document.querySelector('#playlistSelect');
|
||||
|
||||
downloadBtn.addEventListener('click', () => {
|
||||
const chosenId = select.value;
|
||||
if (!chosenId) {
|
||||
alert('no playlist selected!');
|
||||
return;
|
||||
}
|
||||
window.location.href = '/download-playlist?playlistId=' + chosenId;
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
`;
|
||||
res.send(html);
|
||||
} catch (err) {
|
||||
console.error('error fetching playlists:', err);
|
||||
res.status(500).send('error fetching playlists.');
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* called when the user has selected a playlist from the popup
|
||||
* fetch all items, write them to a json file, and update the status
|
||||
*/
|
||||
app.get('/download-playlist', async (req, res) => {
|
||||
try {
|
||||
if (!oauth2Client.credentials || !oauth2Client.credentials.access_token) {
|
||||
return res
|
||||
.status(401)
|
||||
.send('error: oauth2 client not authorized. go to /auth first.');
|
||||
}
|
||||
|
||||
const { playlistId } = req.query;
|
||||
if (!playlistId) {
|
||||
return res
|
||||
.status(400)
|
||||
.send('missing playlist id. please choose a playlist.');
|
||||
}
|
||||
|
||||
// set status to in-progress
|
||||
downloadStatus = 'in-progress';
|
||||
|
||||
// fetch the playlist items
|
||||
const items = await getPlaylistItems(playlistId, oauth2Client);
|
||||
|
||||
// create a data folder if it doesn't exist
|
||||
const dataDir = path.join(process.cwd(), 'data');
|
||||
if (!fs.existsSync(dataDir)) {
|
||||
fs.mkdirSync(dataDir);
|
||||
}
|
||||
|
||||
const outFile = path.join(dataDir, `playlist_${playlistId}.json`);
|
||||
|
||||
fs.writeFileSync(outFile, JSON.stringify(items, null, 2), 'utf8');
|
||||
|
||||
downloadStatus = 'completed';
|
||||
|
||||
res.send(`
|
||||
<html>
|
||||
<head><title>download complete</title></head>
|
||||
<body>
|
||||
<h1>download complete!</h1>
|
||||
<p>downloaded ${items.length} items to <strong>${outFile}</strong></p>
|
||||
<p><a href="/status" target="_blank">check status</a></p>
|
||||
<script>window.close()</script>
|
||||
</body>
|
||||
</html>
|
||||
`);
|
||||
} catch (err) {
|
||||
console.error('error downloading playlist:', err);
|
||||
downloadStatus = 'error';
|
||||
res.status(500).send('error downloading playlist.');
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
app.get('/status', (_req, res) => {
|
||||
let html = `
|
||||
<html>
|
||||
<head>
|
||||
<title>download status</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>current status: ${downloadStatus}</h1>
|
||||
</body>
|
||||
</html>
|
||||
`;
|
||||
res.send(html);
|
||||
});
|
||||
|
||||
|
||||
//#endregion
|
||||
|
||||
|
||||
app.listen(port, () => {
|
||||
console.log(`server listening on http://localhost:${port}`);
|
||||
console.log(`go to http://localhost:${port}/auth to start oauth flow`);
|
||||
});
|
||||
@@ -0,0 +1,54 @@
|
||||
import { google } from "googleapis";
|
||||
import fs from 'fs';
|
||||
import { tokenManager } from "./tokenManager.js";
|
||||
(await import('dotenv')).config({
|
||||
path: './secret/config.env',
|
||||
debug: true
|
||||
});
|
||||
|
||||
const { CLIENT_ID, CLIENT_SECRET, REDIRECT_URI } = process.env,
|
||||
manager = new tokenManager({ clientId: CLIENT_ID, clientSecret: CLIENT_SECRET, redirectUri: REDIRECT_URI, tokenPath: 'secret/token.json' });
|
||||
|
||||
if (!manager.loadToken()) throw 'LOAD TOKEN FAILED!';
|
||||
|
||||
const youtube = google.youtube('v3'),
|
||||
video = await youtube.videos.list({
|
||||
auth: manager.getAuthClient(),
|
||||
part: 'snippet,contentDetails',
|
||||
myRating: 'like',
|
||||
maxResults: 1,
|
||||
// pageToken: nextPageToken
|
||||
});
|
||||
|
||||
const channelsinfo = (await (youtube.channels.list({ auth: manager.getAuthClient(), mine: true, part: 'snippet,contentDetails,statistics' }))).data;
|
||||
fs.writeFileSync('channels.json', JSON.stringify(channelsinfo));
|
||||
|
||||
let likedMusic = [];
|
||||
let nextPageToken = null;
|
||||
|
||||
// first, retrieve *all* liked videos
|
||||
do {
|
||||
const response = await youtube.videos.list({
|
||||
auth: manager.getAuthClient(),
|
||||
part: 'snippet,contentDetails',
|
||||
myRating: 'like',
|
||||
maxResults: 50,
|
||||
pageToken: nextPageToken
|
||||
});
|
||||
|
||||
if (response.data.items) {
|
||||
likedMusic = likedMusic.concat(response.data.items.filter(o => o.snippet?.categoryId === '10').map(o => o.snippet.title))
|
||||
// snippet.categoryId should be present under `video.snippet`
|
||||
const t = response.data.items.find(video => video.snippet.title === 'Peeping Tom (feat. Rosie Harte)')
|
||||
if (t) {
|
||||
fs.writeFileSync('temp.json', JSON.stringify(t));
|
||||
break;
|
||||
}
|
||||
// likedMusic = likedMusic.concat(response.data.items.filter((video) => video.snippet?.categoryId === '10'));
|
||||
}
|
||||
|
||||
nextPageToken = response.data.nextPageToken;
|
||||
} while (nextPageToken);
|
||||
|
||||
// console.log('not found!');
|
||||
fs.writeFileSync('temp.json', JSON.stringify(likedMusic))
|
||||
@@ -0,0 +1,94 @@
|
||||
// APPARENTLY the youtube api just....doesn't return all likes for some reason.....
|
||||
|
||||
import { chromium } from 'playwright';
|
||||
import fs from 'fs';
|
||||
import dotenv from 'dotenv';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
const urltostr = (u) => {
|
||||
try {
|
||||
return new URL(u);
|
||||
}
|
||||
catch (err) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function scrapeLikedVideos() {
|
||||
const browser = await chromium.launchPersistentContext('bdata', {
|
||||
headless: false, // youtube breaks in headless
|
||||
args: ['--disable-blink-features=AutomationControlled']
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
|
||||
console.log("Opening YouTube...");
|
||||
await page.goto('https://music.youtube.com/', { waitUntil: 'networkidle' });
|
||||
|
||||
// Step Log in or die
|
||||
if (await page.locator('[aria-label="Sign in"]').isVisible()) {
|
||||
console.log("Logging in...");
|
||||
|
||||
await page.click('[aria-label="Sign in"]');
|
||||
await page.waitForNavigation({ waitUntil: 'networkidle' });
|
||||
|
||||
console.log(page.url());
|
||||
await page.waitForURL('https://music.youtube.com/').catch(console.error);
|
||||
|
||||
console.log("Login successful");
|
||||
} else {
|
||||
console.log("Already logged in");
|
||||
}
|
||||
|
||||
// Navigate to "Liked Videos" playlist
|
||||
console.log("Navigating to Liked Videos...");
|
||||
await page.goto('https://music.youtube.com/playlist?list=LM', { waitUntil: 'domcontentloaded' });
|
||||
|
||||
// Scroll to load all liked videos
|
||||
console.log("Scrolling through Liked Videos...");
|
||||
const s = new Set();
|
||||
let previousHeight = 0;
|
||||
while (true) {
|
||||
const currentHeight = await page.evaluate(() => document.body.scrollHeight);
|
||||
if (currentHeight === previousHeight) break;
|
||||
|
||||
previousHeight = currentHeight;
|
||||
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
|
||||
await page.waitForTimeout(2000); // Wait for new content to load
|
||||
|
||||
// sloppy and repetative to do it every time, but otherwise it won't work as the incoming videos won't all appear
|
||||
(await page.evaluate(() => {
|
||||
const videos = Array.from(document.querySelector('#contents').querySelectorAll('.title .yt-simple-endpoint'));
|
||||
return videos.map(video => video.href.replace('&list=LM', ''));
|
||||
})).map(u => s.add(u));
|
||||
}
|
||||
|
||||
// // Scrape video data
|
||||
// console.log("Scraping liked videos...");
|
||||
// const likedVideos = await page.evaluate(() => {
|
||||
// const videos = Array.from(document.querySelector('#contents').querySelectorAll('.title .yt-simple-endpoint'));
|
||||
// return videos.map(video => video.href);
|
||||
// });
|
||||
|
||||
// console.log(`Found ${likedVideos.length} liked videos.`);
|
||||
// console.log(likedVideos);
|
||||
|
||||
// Close the browser
|
||||
await browser.close();
|
||||
|
||||
// Save the results to a JSON file
|
||||
fs.writeFileSync('liked_videos.json', JSON.stringify([...s], null, 2));
|
||||
console.log("Liked videos saved to liked_videos.json");
|
||||
|
||||
return [...s];
|
||||
}
|
||||
|
||||
// Run the scraper
|
||||
(async () => {
|
||||
try {
|
||||
await scrapeLikedVideos();
|
||||
} catch (error) {
|
||||
console.error("Error scraping liked videos:", error);
|
||||
}
|
||||
})();
|
||||
@@ -0,0 +1,61 @@
|
||||
import fs from 'fs';
|
||||
import { google } from 'googleapis';
|
||||
|
||||
|
||||
export class tokenManager {
|
||||
constructor({
|
||||
clientId,
|
||||
clientSecret,
|
||||
redirectUri,
|
||||
tokenPath = 'token.json'
|
||||
}) {
|
||||
// store options
|
||||
this.clientId = clientId;
|
||||
this.clientSecret = clientSecret;
|
||||
this.redirectUri = redirectUri;
|
||||
this.tokenPath = tokenPath;
|
||||
|
||||
// create oauth2 client
|
||||
this.oauth2Client = new google.auth.OAuth2(
|
||||
this.clientId,
|
||||
this.clientSecret,
|
||||
this.redirectUri
|
||||
);
|
||||
}
|
||||
|
||||
loadToken() {
|
||||
if (!fs.existsSync(this.tokenPath)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const tokenData = fs.readFileSync(this.tokenPath, 'utf-8');
|
||||
const token = JSON.parse(tokenData);
|
||||
this.oauth2Client.setCredentials(token);
|
||||
|
||||
return token;
|
||||
}
|
||||
|
||||
|
||||
saveToken(token) {
|
||||
fs.writeFileSync(this.tokenPath, JSON.stringify(token, null, 2), 'utf-8');
|
||||
this.oauth2Client.setCredentials(token);
|
||||
}
|
||||
|
||||
async refreshAccessToken() {
|
||||
// if no refresh token is present, we can't refresh
|
||||
if (!this.oauth2Client.credentials.refresh_token) {
|
||||
throw new Error('no refresh token is available');
|
||||
}
|
||||
|
||||
// use the googleapis refresh method
|
||||
const { credentials } = await this.oauth2Client.refreshAccessToken();
|
||||
|
||||
// save the new token info
|
||||
this.saveToken(credentials);
|
||||
return credentials;
|
||||
}
|
||||
|
||||
getAuthClient() {
|
||||
return this.oauth2Client;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user