v1.3.0: add --parser.max-pages and Google Sheets real-time writer

.github/workflows/release.yml

@@ -12,42 +12,39 @@ jobs:
     strategy:
       matrix:
         include:
-          - os: ubuntu-20.04
+          - os: ubuntu-22.04
             dist: linux
 
-          - os: macos-12
+          - os: macos-13
             dist: macos
             ext:
 
           - os: macos-14
             dist: macos-arm
             ext:
 
-          - os: windows-2019
+          - os: windows-2022
             dist: windows
 
     runs-on: ${{ matrix.os }}
 
     steps:
     - name: Get the version
       id: get_version
-      run: echo ::set-output name=version::${GITHUB_REF#refs/tags/}
+      run: echo "version=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
       shell: bash
 
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up Python 3.11
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v5
       with:
         python-version: "3.11.3"
 
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         python -m pip install -e .[gui,dev]
-        # Replace pydantic with no-binary version to lightweight target binary a bit
-        # python -m pip uninstall -y pydantic
-        # python -m pip install --no-binary pydantic pydantic>=1.9.0
 
     - name: Build standalone app
       run: |
@@ -64,7 +61,7 @@ jobs:
       shell: bash
 
     - name: Upload distributive
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: dist-${{ matrix.dist }}
         path: Parser2GIS-*
@@ -81,10 +78,10 @@ jobs:
     - name: Get the version
       id: get_version
       run: |
-        echo ::set-output name=version::${GITHUB_REF#refs/tags/v}
+        echo "version=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
       shell: bash
 
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Get Changelog Entry
       id: changelog_reader
@@ -93,7 +90,7 @@ jobs:
         version: ${{ steps.get_version.outputs.version }}
         path: ./CHANGELOG.md
 
-    - uses: actions/download-artifact@v3
+    - uses: actions/download-artifact@v4
 
     - uses: ncipollo/release-action@v1.10.0
       with:
@@ -104,4 +101,4 @@ jobs:
         draft: true
         artifacts: "dist-*/*"
         allowUpdates: true
-        token: ${{ secrets.GITHUB_TOKEN }}
+        token: ${{ secrets.GITHUB_TOKEN }}

CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## [Невошедшее]
 
+## [1.3.0] - 06-07-2026
+### Добавлено
+- Добавлен параметр `--parser.max-pages` — ограничение количества страниц с одного URL (по умолчанию: 0 = без лимита). Можно указать, например, `--parser.max-pages 60`.
+- Добавлена запись спарсенных данных напрямую в Google Sheets через `--gsheets-id KEY`. Данные записываются батчами по 50 записей, что предотвращает потерю данных при сбое. Требуется service account JSON файл (указывается через `--gsheets-credentials PATH` или переменную `PARSER_2GIS_GOOGLE_CREDENTIALS`).
+
 ## [1.2.1] - 14-03-2024
 ### Добавлено
 - Добавлена поддержка парсинга остановок. Fix [issue](https://github.com/interlark/parser-2gis/issues/52).
@@ -95,6 +100,7 @@
 
 
 [Невошедшее]: https://github.com/interlark/parser-2gis/compare/v1.2.1...HEAD
+[1.3.0]: https://github.com/interlark/parser-2gis/compare/v1.2.1...v1.3.0
 [1.2.1]: https://github.com/interlark/parser-2gis/compare/v1.2.0...v1.2.1
 [1.2.0]: https://github.com/interlark/parser-2gis/compare/v1.1.2...v1.2.0
 [1.1.2]: https://github.com/interlark/parser-2gis/compare/v1.1.1...v1.1.2

parser_2gis/config.py

@@ -21,6 +21,8 @@ class Configuration(BaseModel):
     writer: WriterOptions = WriterOptions()
     chrome: ChromeOptions = ChromeOptions()
     parser: ParserOptions = ParserOptions()
+    gsheets_id: Optional[str] = None
+    gsheets_credentials: Optional[str] = None
     path: Optional[pathlib.Path] = None
     version: str = config_version
 

parser_2gis/main.py

@@ -122,11 +122,16 @@ def parse_arguments() -> tuple[argparse.Namespace, Configuration]:
     p_parser.add_argument('--parser.max-records', metavar='{1000,2000,...}', help='Максимальное количество спарсенных записей с одного URL')
     p_parser.add_argument('--parser.skip-404-response', metavar='{yes,no}', help='Пропускать ссылки вернувшие сообщение "Точных совпадений нет / Не найдено"')
     p_parser.add_argument('--parser.delay_between_clicks', metavar='{0,100,...}', help='Задержка между кликами по записям (миллисекунд)')
+    p_parser.add_argument('--parser.max-pages', metavar='{0,10,20,...}', help='Максимальное количество страниц с одного URL (0 = без лимита)')
 
     other_parser = arg_parser.add_argument_group('Прочие аргументы')
     other_parser.add_argument('--writer.verbose', metavar='{yes,no}', help='Отображать наименования позиций во время парсинга')
     other_parser.add_argument('--writer.encoding', metavar='{utf8,1251,...}', help='Кодировка результирующего файла')
 
+    gsheets_parser = arg_parser.add_argument_group('Google Sheets')
+    gsheets_parser.add_argument('--gsheets-id', metavar='KEY', help='ID Google Sheets таблицы для параллельной записи')
+    gsheets_parser.add_argument('--gsheets-credentials', metavar='PATH', help='Путь к service account JSON файлу (по умолчанию: переменная PARSER_2GIS_GOOGLE_CREDENTIALS)')
+
     rest_parser = arg_parser.add_argument_group('Служебные аргументы')
     rest_parser.add_argument('-v', '--version', action='version', version=f'%(prog)s {version}', help='Показать версию программы и выйти')
     rest_parser.add_argument('-h', '--help', action='help', help='Показать эту справку и выйти')
@@ -166,4 +171,4 @@ def main() -> None:
         config = command_line_config
         app = cli_app
 
-    app(args.url, args.output_path, args.format, config)
+    app(args.url, args.output_path, args.format, config)

parser_2gis/parser/options.py

@@ -19,11 +19,13 @@ class ParserOptions(BaseModel):
         skip_404_response: Whether to skip 404 document response or not.
         delay_between_clicks: Delay between each item's click in milliseconds.
         max_records: Max number of records to parse from one URL.
+        max_pages: Max number of pages to parse (0 = unlimited).
         use_gc: Use Garbage Collector.
         gc_pages_interval: Run Garbage Collector every N pages (if `use_gc` enabled).
     """
     skip_404_response: bool = True
     delay_between_clicks: NonNegativeInt = 0
     max_records: PositiveInt = default_max_records()
+    max_pages: NonNegativeInt = 0
     use_gc: bool = False
     gc_pages_interval: PositiveInt = 10

parser_2gis/parser/parsers/main.py

@@ -269,6 +269,11 @@ def get_unique_links() -> list[DOMNode]:
             if walk_page_number and walk_page_number <= current_page_number:
                 walk_page_number = None
 
+            # Limit pages if max_pages is set
+            if self._options.max_pages and current_page_number >= self._options.max_pages:
+                logger.info('Достигнут лимит страниц (%d).', self._options.max_pages)
+                return
+
     def close(self) -> None:
         self._chrome_remote.stop()
 

parser_2gis/runner/cli.py

@@ -3,7 +3,7 @@
 from ..exceptions import ChromeRuntimeException, ChromeUserAbortException
 from ..logger import logger
 from ..parser import get_parser
-from ..writer import get_writer
+from ..writer import get_writer, GoogleSheetsWriter
 from .runner import AbstractRunner
 
 
@@ -18,15 +18,44 @@ class CLIRunner(AbstractRunner):
     """
     def start(self):
         logger.info('Парсинг запущен.')
+
+        gsheets_writer = None
         try:
             with get_writer(self._output_path, self._format, self._config.writer) as writer:
+                # Also write to Google Sheets if configured
+                if self._config.gsheets_id:
+                    try:
+                        gsheets_writer = GoogleSheetsWriter(
+                            self._config.gsheets_id,
+                            self._config.writer,
+                            self._config.gsheets_credentials,
+                        )
+                        gsheets_writer.__enter__()
+                        logger.info('Google Sheets запись включена (таблица: %s)', self._config.gsheets_id)
+                    except Exception as e:
+                        logger.error('Ошибка инициализации Google Sheets: %s', e)
+
+                # Wrap writers into a single writer interface
+                class _MultiWriter:
+                    """Writes to multiple writers simultaneously."""
+                    def __init__(self, writers):
+                        self._writers = writers
+                    def write(self, doc):
+                        for w in self._writers:
+                            w.write(doc)
+
+                writers = [writer]
+                if gsheets_writer:
+                    writers.append(gsheets_writer)
+                multi_writer = _MultiWriter(writers)
+
                 for url in self._urls:
                     logger.info(f'Парсинг ссылки {url}')
                     with get_parser(url,
                                     chrome_options=self._config.chrome,
                                     parser_options=self._config.parser) as parser:
                         try:
-                            parser.parse(writer)
+                            parser.parse(multi_writer)
                         finally:
                             logger.info('Парсинг ссылки завершён.')
         except (KeyboardInterrupt, ChromeUserAbortException):
@@ -37,7 +66,12 @@ def start(self):
             else:
                 logger.error('Ошибка во время работы парсера.', exc_info=True)
         finally:
+            if gsheets_writer:
+                try:
+                    gsheets_writer.__exit__(None, None, None)
+                except Exception:
+                    pass
             logger.info('Парсинг завершён.')
 
     def stop(self):
-        pass
+        pass

parser_2gis/version.py

@@ -1,4 +1,4 @@
 """Version info."""
 
-version = '1.2.1'
+version = '1.3.0'
 config_version = '0.1'

parser_2gis/writer/__init__.py

@@ -1,5 +1,5 @@
 from .options import WriterOptions, CSVOptions
-from .writers import CSVWriter, JSONWriter, FileWriter, XLSXWriter
+from .writers import CSVWriter, JSONWriter, FileWriter, XLSXWriter, GoogleSheetsWriter
 from .factory import get_writer
 
 __all__ = [
@@ -9,5 +9,6 @@
     'XLSXWriter',
     'JSONWriter',
     'FileWriter',
+    'GoogleSheetsWriter',
     'get_writer',
 ]

parser_2gis/writer/factory.py

@@ -2,7 +2,7 @@
 
 from typing import TYPE_CHECKING
 
-from .writers import CSVWriter, XLSXWriter, FileWriter, JSONWriter
+from .writers import CSVWriter, XLSXWriter, FileWriter, JSONWriter, GoogleSheetsWriter
 
 from .exceptions import WriterUnknownFileFormat
 

parser_2gis/writer/writers/__init__.py

@@ -2,10 +2,12 @@
 from .csv_writer import CSVWriter
 from .json_writer import JSONWriter
 from .xlsx_writer import XLSXWriter
+from .gsheets_writer import GoogleSheetsWriter
 
 __all__ = [
     'FileWriter',
     'CSVWriter',
     'XLSXWriter',
     'JSONWriter',
+    'GoogleSheetsWriter',
 ]