transformer/src/cloudimagedirectory/transformer.py at 4b318b7d8d6a14a10557de9d9c911ba7e3edae11 · redhatcloudx/transformer · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
"""Add command for transforming image data."""
import datetime

import click
import pandas as pd

from cloudimagedirectory.connection import connection
from cloudimagedirectory.filter import filter
from cloudimagedirectory.transform import transform


@click.command()
@click.option(
    "-op",
    "--origin.path",
    "origin_path",
    prompt="Origin filesystem path",
    help="The path to the filesystem source",
)
@click.option(
    "-dp",
    "--destination.path",
    "destination_path",
    prompt="Destination filesystem path",
    help="The path to the filesystem destination",
)
@click.option(
    "-fu",
    "--filter.until",
    "filter_until",
    prompt="Ignore all images after timestamp e.g. (YYYY-MM-DD)",
    default="none",
    help="Ignores all images after given timestamp",
)
@click.option(
    "-f",
    "--input.files",
    "arg_files",
    default="none",
    prompt="files to process",
    help="List of predefined files to process",
)
def run(
    origin_path: str, destination_path: str, arg_files: str, filter_until: str
) -> None:
    """Get content from filesystem format image data."""
    target: list[str] = []
    if arg_files != "none":
        target = arg_files.split(",")
    origin_connection = connection.ConnectionFS(origin_path, target)
    filenames = origin_connection.get_filenames()
    for file in filenames:
        print("input: " + file.filename)

    filters = [
        filter.FilterImageByFilename("test"),
        filter.FilterImageByFilename("beta"),
        filter.FilterImageByUniqueReference(),
    ]

    if filter_until == "default":
        # NOTE: Subtract 2 years from current time
        filter_after = datetime.datetime.now() - datetime.timedelta(days=2 * 365)
        filters.append(filter.FilterImageByLatestUpdate(filter_after))
    elif filter_until != "" and filter_until != "none":
        filter_after = pd.to_datetime(filter_until)
        filters.append(filter.FilterImageByLatestUpdate(filter_after))

    pipeline = transform.Pipeline(
        origin_connection,
        [
            transform.TransformerAWS,
            transform.TransformerAZURE,
            transform.TransformerGoogle,
        ],
        filters,
        [
            transform.TransformerIdxListImageNames,
            transform.TransformerIdxListImageLatest,
            transform.TransformerIdxListImageLatestGoogle,
            transform.TransformerIdxListImageLatestAWS,
            transform.TransformerIdxListImageLatestAZURE,
            transform.TransformerV2All,
            transform.TransformerV2ListOS,
        ],
    )
    print("run pipeline")
    results = pipeline.run(filenames)

    for result in results:
        result.filename = destination_path + "/" + result.filename
        if not result.is_raw():
            origin_connection.put_content(result)