refactor scanner, add progress bar
This commit is contained in:
evilchili
@@ -1,9 +1,9 @@
|
||||
import os
|
||||
|
||||
from prompt_toolkit.completion import Completion, FuzzyCompleter
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
import groove.path
|
||||
|
||||
from . import metadata
|
||||
|
||||
|
||||
@@ -37,7 +37,8 @@ class DatabaseManager:
|
||||
@property
|
||||
def engine(self):
|
||||
if not self._engine:
|
||||
self._engine = create_engine(f"sqlite:///{os.environ.get('DATABASE_PATH')}?check_same_thread=False", future=True)
|
||||
path = groove.path.database()
|
||||
self._engine = create_engine(f"sqlite:///{path}?check_same_thread=False", future=True)
|
||||
return self._engine
|
||||
|
||||
@property
|
||||
|
||||
+160
-29
@@ -1,48 +1,111 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import music_tag
|
||||
|
||||
from itertools import chain
|
||||
from pathlib import Path
|
||||
from typing import Callable, Union, Iterable
|
||||
|
||||
import music_tag
|
||||
import rich.repr
|
||||
|
||||
from rich.console import Console
|
||||
from rich.progress import (
|
||||
Progress,
|
||||
TextColumn,
|
||||
BarColumn,
|
||||
SpinnerColumn,
|
||||
TimeRemainingColumn
|
||||
)
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy.exc import NoResultFound
|
||||
|
||||
import groove.db
|
||||
import groove.path
|
||||
|
||||
from groove.exceptions import InvalidPathError
|
||||
|
||||
|
||||
@rich.repr.auto(angular=True)
|
||||
class MediaScanner:
|
||||
"""
|
||||
Scan a directory structure containing audio files and import them into the database.
|
||||
SYNOPSIS
|
||||
|
||||
Scan a directory structure containing audio files and import track entries
|
||||
into the Groove on Demand database. Existing tracks will be ignored.
|
||||
|
||||
USAGE
|
||||
|
||||
MediaScanner(db=DB, [ARGS])
|
||||
|
||||
ARGS
|
||||
|
||||
db An sqlalchemy databse session
|
||||
console A rich console instance
|
||||
glob A pattern to search for. Defaults to MEDIA_GLOB. Multiple
|
||||
patterns can be specifed as a comma-separated-list.
|
||||
path The path to scan. Defaults to MEDIA_ROOT.
|
||||
root The media root, as specified by MEDIA_ROOT
|
||||
|
||||
EXAMPLES
|
||||
|
||||
MediaScanner(db=DB, path='Kid Koala', glob='*.mp3').scan()
|
||||
>>> 15
|
||||
|
||||
INSTANCE ATTRIBUTES
|
||||
|
||||
db The databse session
|
||||
console The rich console instance
|
||||
glob The globs to search for
|
||||
path The path to be scanned
|
||||
root The media root
|
||||
|
||||
"""
|
||||
def __init__(self, root: Union[Path, None], db: Callable, glob: Union[str, None] = None) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
db: Callable,
|
||||
path: Union[Path, None] = None,
|
||||
glob: Union[str, None] = None,
|
||||
console: Union[Console, None] = None,
|
||||
) -> None:
|
||||
self._db = db
|
||||
self._glob = tuple((glob or os.environ.get('MEDIA_GLOB')).split(','))
|
||||
self._root = root or groove.path.media_root()
|
||||
logging.debug(f"Configured media scanner for root: {self._root}")
|
||||
self._root = groove.path.media_root()
|
||||
self._console = console or Console()
|
||||
self._scanned = 0
|
||||
self._imported = 0
|
||||
self._total = 0
|
||||
self._path = self._configure_path(path)
|
||||
|
||||
@property
|
||||
def db(self) -> Callable:
|
||||
return self._db
|
||||
|
||||
@property
|
||||
def console(self) -> Console:
|
||||
return self._console
|
||||
|
||||
@property
|
||||
def root(self) -> Path:
|
||||
return self._root
|
||||
|
||||
@property
|
||||
def path(self) -> Path:
|
||||
return self._path
|
||||
|
||||
@property
|
||||
def glob(self) -> tuple:
|
||||
return self._glob
|
||||
|
||||
def find_sources(self, pattern):
|
||||
return self.root.rglob(pattern) # pragma: no cover
|
||||
|
||||
def import_tracks(self, sources: Iterable) -> None:
|
||||
async def _do_import():
|
||||
logging.debug("Scanning filesystem (this may take a minute)...")
|
||||
for path in sources:
|
||||
asyncio.create_task(self._import_one_track(path))
|
||||
asyncio.run(_do_import())
|
||||
self.db.commit()
|
||||
def _configure_path(self, path):
|
||||
if not path: # pragma: no cover
|
||||
return self._root
|
||||
fullpath = Path(self._root) / Path(path)
|
||||
if not (fullpath.exists() and fullpath.is_dir()):
|
||||
raise InvalidPathError( # pragma: no cover
|
||||
f"[b]{fullpath}[/b] does not exist or is not a directory."
|
||||
)
|
||||
return fullpath
|
||||
|
||||
def _get_tags(self, path): # pragma: no cover
|
||||
tags = music_tag.load_file(path)
|
||||
@@ -51,12 +114,83 @@ class MediaScanner:
|
||||
'title': str(tags['title']),
|
||||
}
|
||||
|
||||
async def _import_one_track(self, path):
|
||||
tags = self._get_tags(path)
|
||||
tags['relpath'] = str(path.relative_to(self.root))
|
||||
stmt = groove.db.track.insert(tags).prefix_with('OR IGNORE')
|
||||
logging.debug(f"{tags['artist']} - {tags['title']}")
|
||||
self.db.execute(stmt)
|
||||
def find_sources(self, pattern):
|
||||
"""
|
||||
Recursively search the instance path for files matching the pattern.
|
||||
"""
|
||||
entrypoint = self._path if self._path else self._root
|
||||
for path in entrypoint.rglob(pattern): # pragma: no cover
|
||||
if not path.is_dir():
|
||||
yield path
|
||||
|
||||
def import_tracks(self, sources: Iterable) -> None:
|
||||
"""
|
||||
Step through the specified source files and schedule async tasks to
|
||||
import them, reporting progress via a rich progress bar.
|
||||
"""
|
||||
|
||||
async def _do_import(progress, scanner):
|
||||
tasks = set()
|
||||
for path in sources:
|
||||
self._total += 1
|
||||
progress.update(scanner, total=self._total)
|
||||
tasks.add(asyncio.create_task(
|
||||
self._import_one_track(path, progress, scanner)))
|
||||
progress.start_task(scanner)
|
||||
|
||||
progress = Progress(
|
||||
TimeRemainingColumn(compact=True, elapsed_when_finished=True),
|
||||
BarColumn(bar_width=15),
|
||||
TextColumn("[progress.percentage]{task.percentage:>3.0f}%", justify="left"),
|
||||
TextColumn("[dim]|"),
|
||||
TextColumn("[title]{task.total:-6d}[/title] [b]total", justify="right"),
|
||||
TextColumn("[dim]|"),
|
||||
TextColumn("[title]{task.fields[imported]:-6d}[/title] [b]new", justify="right"),
|
||||
TextColumn("[dim]|"),
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
console=self.console,
|
||||
)
|
||||
with progress:
|
||||
scanner = progress.add_task(
|
||||
f"[bright]Scanning [link]{self.path}[/link] (this may take some time)...",
|
||||
imported=0,
|
||||
total=0,
|
||||
start=False
|
||||
)
|
||||
asyncio.run(_do_import(progress, scanner))
|
||||
progress.update(
|
||||
scanner,
|
||||
completed=self._total,
|
||||
description=f"[bright]Scan of [link]{self.path}[/link] complete!",
|
||||
)
|
||||
|
||||
async def _import_one_track(self, path, progress, scanner):
|
||||
"""
|
||||
Import a single audo file into the databse, unless it already exists.
|
||||
"""
|
||||
self._scanned += 1
|
||||
relpath = str(path.relative_to(self.root))
|
||||
try:
|
||||
self.db.query(groove.db.track).filter(
|
||||
groove.db.track.c.relpath == relpath).one()
|
||||
return
|
||||
except NoResultFound:
|
||||
pass
|
||||
|
||||
columns = self._get_tags(path)
|
||||
columns['relpath'] = relpath
|
||||
|
||||
logging.debug(f"Importing: {columns}")
|
||||
self.db.execute(groove.db.track.insert(columns))
|
||||
self.db.commit()
|
||||
self._imported += 1
|
||||
progress.update(
|
||||
scanner,
|
||||
imported=self._imported,
|
||||
completed=self._scanned,
|
||||
description=f"[bright]Imported [artist]{columns['artist']}[/artist]: [title]{columns['title']}[/title]",
|
||||
)
|
||||
|
||||
def scan(self) -> int:
|
||||
"""
|
||||
@@ -64,12 +198,9 @@ class MediaScanner:
|
||||
found. Existing entries will be ignored.
|
||||
"""
|
||||
count = self.db.query(func.count(groove.db.track.c.relpath)).scalar()
|
||||
logging.debug(f"Track table currently contains {count} entries.")
|
||||
for pattern in self.glob:
|
||||
self.import_tracks(self.find_sources(pattern))
|
||||
newcount = self.db.query(func.count(groove.db.track.c.relpath)).scalar() - count
|
||||
logging.debug(f"Inserted {newcount} new tracks so far this run...")
|
||||
combined_sources = chain.from_iterable(
|
||||
self.find_sources(pattern) for pattern in self.glob
|
||||
)
|
||||
self.import_tracks(combined_sources)
|
||||
newcount = self.db.query(func.count(groove.db.track.c.relpath)).scalar() - count
|
||||
return newcount
|
||||
|
||||
|
||||
media_scanner = MediaScanner
|
||||
|
||||
Reference in New Issue
Block a user