feat: languages filter (closes #59)

Author: Justintime50

CHANGELOG.md

@@ -1,5 +1,10 @@
 # CHANGELOG
 
+## v6.3.0 (2025-03-13)
+
+- Adds new `languages` CLI flag allowing filtering via a comma-separated list of GitHub languages for repos (cannot be used with include/exclude) - (closes #59)
+- Forking repos is now a threaded operation like all other operations instead of sequential
+
 ## v6.2.0 (2025-03-13)
 
 - Forking gists is now a threaded operation like all other operations instead of sequential

README.md

@@ -50,6 +50,7 @@ Options:
     -f, --fork            Pass this flag to fork git assets.
     --include INCLUDE     Pass a comma separated list of repos to filter what is included in the Archive.
     --exclude EXCLUDE     Pass a comma separated list of repos to filter what is excluded from the Archive.
+    --languages LANGUAGES Pass a comma separated list of languages to filter what is included in the Archive.
     --forks               Pass this flag to include forked git assets (when cloning or pulling).
     --location LOCATION   The location where you want your GitHub Archive to be stored. Default: /Users/USERNAME/github-archive
     --https               Use HTTPS URLs instead of SSH.

github_archive/_version.py

@@ -1 +1 @@
-__version__ = "6.2.0"
+__version__ = "6.3.0"

github_archive/archive.py

@@ -60,6 +60,7 @@ def __init__(
         fork=False,
         include=None,
         exclude=None,
+        languages=None,
         forks=False,
         location=DEFAULT_LOCATION,
         use_https=False,
@@ -80,6 +81,7 @@ def __init__(
         self.fork = fork
         self.include = include.lower().split(',') if include else ''
         self.exclude = exclude.lower().split(',') if exclude else ''
+        self.languages = languages.lower().split(',') if languages else ''
         self.forks = forks
         self.location = os.path.expanduser(location)
         self.use_https = use_https
@@ -147,7 +149,7 @@ def run(self):
                 _ = iterate_repos_to_archive(self, user_repos, PULL_OPERATION)
             if self.fork:
                 logger.info('# Forking user repos...')
-                iterate_repos_to_fork(user_repos)
+                iterate_repos_to_fork(self, user_repos)
 
         # Orgs
         if self.orgs:
@@ -167,7 +169,7 @@ def run(self):
                 _ = iterate_repos_to_archive(self, org_repos, PULL_OPERATION)
             if self.fork:
                 logger.info('# Forking org repos...')
-                iterate_repos_to_fork(org_repos)
+                iterate_repos_to_fork(self, org_repos)
 
         # Stars
         if self.stars:
@@ -187,7 +189,7 @@ def run(self):
                 _ = iterate_repos_to_archive(self, starred_repos, PULL_OPERATION)
             if self.fork:
                 logger.info('# Forking starred repos...')
-                iterate_repos_to_fork(starred_repos)
+                iterate_repos_to_fork(self, starred_repos)
 
         if failed_repo_dirs:
             logger.info('Cleaning up repos...')
@@ -261,6 +263,11 @@ def initialize_project(self):
                 logger=logger,
                 message='The include and exclude flags are mutually exclusive. Only one can be used on each run.',
             )
+        elif (self.include or self.exclude) and self.languages:
+            log_and_raise_value_error(
+                logger=logger,
+                message='The include and exclude flags cannot be used with the languages flag.',
+            )
 
     def authenticated_user_in_users(self) -> bool:
         """Returns True if the authenticated user is in the list of users."""

github_archive/cli.py

@@ -109,6 +109,13 @@ def __init__(self):
             default=None,
             help='Pass a comma separated list of repos to filter what is excluded from the Archive.',
         )
+        parser.add_argument(
+            '--languages',
+            type=str,
+            required=False,
+            default=None,
+            help='Pass a comma separated list of languages to filter what is included in the Archive.',
+        )
         parser.add_argument(
             '--forks',
             action='store_true',
@@ -182,6 +189,7 @@ def run(self):
             fork=self.fork,
             include=self.include,
             exclude=self.exclude,
+            languages=self.languages,
             forks=self.forks,
             location=self.location,
             use_https=self.https,

github_archive/gists.py

@@ -28,9 +28,7 @@
 )
 
 
-def iterate_gists_to_archive(
-    github_archive: GithubArchive, gists: List[Gist.Gist], operation: str
-) -> List[Optional[str]]:
+def iterate_gists_to_archive(github_archive: GithubArchive, gists: List[Gist.Gist], operation: str) -> None:
     """Iterate over each gist and start a thread if it can be archived."""
     pool = ThreadPoolExecutor(github_archive.threads)
     thread_list = []
@@ -48,9 +46,6 @@ def iterate_gists_to_archive(
         )
 
     wait(thread_list, return_when=ALL_COMPLETED)
-    failed_gists = [gist.result() for gist in thread_list if gist.result()]
-
-    return failed_gists
 
 
 def view_gists(gists: List[Gist.Gist]):

github_archive/repos.py

@@ -42,7 +42,14 @@ def iterate_repos_to_archive(
 
     for repo in repos:
         if (
-            (not github_archive.include and not github_archive.exclude)
+            (
+                github_archive.languages
+                and repo.language
+                and repo.language.lower() in github_archive.languages
+                and not github_archive.include
+                and not github_archive.exclude
+            )
+            or (not github_archive.languages and not github_archive.include and not github_archive.exclude)
             or (github_archive.include and repo.name in github_archive.include)
             or (github_archive.exclude and repo.name not in github_archive.exclude)
         ):
@@ -58,7 +65,7 @@ def iterate_repos_to_archive(
                 )
             )
         else:
-            logger.debug(f'{repo.name} skipped due to include/exclude filtering')
+            logger.debug(f'{repo.name} skipped due to filtering')
 
     wait(thread_list, return_when=ALL_COMPLETED)
     failed_repos = [repo.result() for repo in thread_list if repo.result()]
@@ -75,10 +82,20 @@ def view_repos(repos: List[Repository.Repository]):
         logger.info(repo_name)
 
 
-def iterate_repos_to_fork(repos: List[Repository.Repository]):
+def iterate_repos_to_fork(github_archive: GithubArchive, repos: List[Repository.Repository]) -> None:
     """Iterates through a list of repos and attempts to fork them."""
+    pool = ThreadPoolExecutor(github_archive.threads)
+    thread_list = []
+
     for repo in repos:
-        _fork_repo(repo)
+        thread_list.append(
+            pool.submit(
+                _fork_repo,
+                repo=repo,
+            )
+        )
+
+    wait(thread_list, return_when=ALL_COMPLETED)
 
 
 def _fork_repo(repo: Repository.Repository):

test/unit/conftest.py

@@ -15,5 +15,6 @@ def mock_git_asset():
     mock_git_asset.owner.login = 'mock_username'
     mock_git_asset.html_url = 'mock/html_url'
     mock_git_asset.ssh_url = 'mock/ssh_url'
+    mock_git_asset.language = 'Python'
 
     return mock_git_asset

test/unit/test_archive.py

@@ -306,6 +306,10 @@ def test_initialize_project(mock_make_dirs, mock_dir_exist, mock_logger):
             {'users': 'justintime50', 'clone': True, 'include': 'mock-repo', 'exclude': 'another-mock-repo'},
             'The include and exclude flags are mutually exclusive. Only one can be used on each run.',
         ),
+        (
+            {'users': 'justintime50', 'clone': True, 'include': 'mock-repo', 'languages': 'python'},
+            'The include and exclude flags cannot be used with the languages flag.',
+        ),
     ],
 )
 @patch('github_archive.archive.Github.get_user')

test/unit/test_repos.py

@@ -1,3 +1,4 @@
+import copy
 import subprocess
 from unittest.mock import (
     MagicMock,
@@ -51,30 +52,51 @@ def test_iterate_repos_matching_authed_username(mock_archive_repo, mock_github_i
 @patch('github_archive.repos._archive_repo')
 def test_iterate_repos_include_list(mock_archive_repo, mock_github_instance, mock_git_asset):
     """Tests that we iterate repos that are on the include list."""
-    repos = [mock_git_asset]
+    mock_non_include_asset = copy.deepcopy(mock_git_asset)
+    mock_non_include_asset.name = 'not-the-name'
+    repos = [mock_git_asset, mock_non_include_asset]
     github_archive = GithubArchive(
         users='mock_username',
         include='mock-asset-name',
     )
 
     iterate_repos_to_archive(github_archive, repos, CLONE_OPERATION)
 
-    mock_archive_repo.assert_called_once()
+    mock_archive_repo.assert_called_once()  # Called once even though there are two, ensure we filtered
 
 
 @patch('github_archive.archive.Github')
 @patch('github_archive.repos._archive_repo')
 def test_iterate_repos_exclude_list(mock_archive_repo, mock_github_instance, mock_git_asset):
     """Tests that we do not iterate repos that are on the exclude list."""
-    repos = [mock_git_asset]
+    mock_non_exclude_asset = copy.deepcopy(mock_git_asset)
+    mock_non_exclude_asset.name = 'not-the-name'
+    repos = [mock_git_asset, mock_non_exclude_asset]
     github_archive = GithubArchive(
         users='mock_username',
         exclude='mock-asset-name',
     )
 
     iterate_repos_to_archive(github_archive, repos, CLONE_OPERATION)
 
-    mock_archive_repo.assert_not_called()
+    mock_archive_repo.assert_called_once()  # Called once even though there are two, ensure we filtered
+
+
+@patch('github_archive.archive.Github')
+@patch('github_archive.repos._archive_repo')
+def test_iterate_repos_languages_list(mock_archive_repo, mock_github_instance, mock_git_asset):
+    """Tests that we iterate repos that are one of the languages in the list."""
+    mock_non_language_asset = copy.deepcopy(mock_git_asset)
+    mock_non_language_asset.language = 'Go'
+    repos = [mock_git_asset, mock_non_language_asset]
+    github_archive = GithubArchive(
+        users='mock_username',
+        languages='python',
+    )
+
+    iterate_repos_to_archive(github_archive, repos, CLONE_OPERATION)
+
+    mock_archive_repo.assert_called_once()  # Called once even though there are two, ensure we filtered
 
 
 @patch('logging.Logger.info')
@@ -157,10 +179,14 @@ def test_archive_repo_called_process_error(mock_logger, mock_subprocess, mock_gi
     mock_logger.assert_called_once()
 
 
+@patch('github_archive.archive.Github')
 @patch('github_archive.repos._fork_repo')
-def test_iterate_repos_to_fork(mock_fork_repo):
+def test_iterate_repos_to_fork(mock_fork_repo, mock_github_instance):
     repo = MagicMock(spec=Repository.Repository)
-    iterate_repos_to_fork([repo])
+    github_archive = GithubArchive(
+        gists='mock_username',
+    )
+    iterate_repos_to_fork(github_archive, [repo])
 
     mock_fork_repo.assert_called_once()