-
Notifications
You must be signed in to change notification settings - Fork 42
Expand file tree
/
Copy pathfeature_store.py
More file actions
1647 lines (1431 loc) · 70.7 KB
/
Copy pathfeature_store.py
File metadata and controls
1647 lines (1431 loc) · 70.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#
# Copyright 2020 Logical Clocks AB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in
# writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import warnings
import datetime
from typing import Optional, Union, List, Dict, TypeVar
import humps
import numpy
import great_expectations as ge
import pandas as pd
import numpy as np
from hsfs.transformation_function import TransformationFunction
from hsfs.client import exceptions
from hsfs.core import transformation_function_engine
from hsfs import (
training_dataset,
feature_group,
feature,
util,
storage_connector,
expectation_suite,
feature_view,
usage,
)
from hsfs.core import (
feature_group_api,
storage_connector_api,
training_dataset_api,
feature_group_engine,
feature_view_engine,
)
from hsfs.constructor.query import Query
from hsfs.statistics_config import StatisticsConfig
class FeatureStore:
DEFAULT_VERSION = 1
def __init__(
self,
featurestore_id,
featurestore_name,
created,
project_name,
project_id,
offline_featurestore_name,
hive_endpoint,
online_enabled,
num_feature_groups=None,
num_training_datasets=None,
num_storage_connectors=None,
num_feature_views=None,
online_featurestore_name=None,
mysql_server_endpoint=None,
online_featurestore_size=None,
**kwargs,
):
self._id = featurestore_id
self._name = featurestore_name
self._created = created
self._project_name = project_name
self._project_id = project_id
self._online_feature_store_name = online_featurestore_name
self._online_feature_store_size = online_featurestore_size
self._offline_feature_store_name = offline_featurestore_name
self._hive_endpoint = hive_endpoint
self._mysql_server_endpoint = mysql_server_endpoint
self._online_enabled = online_enabled
self._num_feature_groups = num_feature_groups
self._num_training_datasets = num_training_datasets
self._num_storage_connectors = num_storage_connectors
self._num_feature_views = num_feature_views
self._feature_group_api = feature_group_api.FeatureGroupApi()
self._storage_connector_api = storage_connector_api.StorageConnectorApi()
self._training_dataset_api = training_dataset_api.TrainingDatasetApi(self._id)
self._feature_group_engine = feature_group_engine.FeatureGroupEngine(self._id)
self._transformation_function_engine = (
transformation_function_engine.TransformationFunctionEngine(self._id)
)
self._feature_view_engine = feature_view_engine.FeatureViewEngine(self._id)
@classmethod
def from_response_json(cls, json_dict):
json_decamelized = humps.decamelize(json_dict)
# fields below are removed from 3.4. remove them for backward compatibility.
json_decamelized.pop("hdfs_store_path", None)
json_decamelized.pop("featurestore_description", None)
json_decamelized.pop("inode_id", None)
return cls(**json_decamelized)
def get_feature_group(self, name: str, version: int = None):
"""Get a feature group entity from the feature store.
Getting a feature group from the Feature Store means getting its metadata handle
so you can subsequently read the data into a Spark or Pandas DataFrame or use
the `Query`-API to perform joins between feature groups.
!!! example
```python
# connect to the Feature Store
fs = ...
fg = fs.get_feature_group(
name="electricity_prices",
version=1,
)
```
# Arguments
name: Name of the feature group to get.
version: Version of the feature group to retrieve, defaults to `None` and will
return the `version=1`.
# Returns
`FeatureGroup`: The feature group metadata object.
# Raises
`hsfs.client.exceptions.RestAPIError`: If unable to retrieve feature group from the feature store.
"""
if version is None:
warnings.warn(
"No version provided for getting feature group `{}`, defaulting to `{}`.".format(
name, self.DEFAULT_VERSION
),
util.VersionWarning,
)
version = self.DEFAULT_VERSION
feature_group_object = self._feature_group_api.get(
self.id, name, version, feature_group_api.FeatureGroupApi.CACHED
)
feature_group_object.feature_store = self
return feature_group_object
def get_feature_groups(self, name: str):
"""Get a list of all versions of a feature group entity from the feature store.
Getting a feature group from the Feature Store means getting its metadata handle
so you can subsequently read the data into a Spark or Pandas DataFrame or use
the `Query`-API to perform joins between feature groups.
!!! example
```python
# connect to the Feature Store
fs = ...
fgs_list = fs.get_feature_groups(
name="electricity_prices"
)
```
# Arguments
name: Name of the feature group to get.
# Returns
`FeatureGroup`: List of feature group metadata objects.
# Raises
`hsfs.client.exceptions.RestAPIError`: If unable to retrieve feature group from the feature store.
"""
feature_group_object = self._feature_group_api.get(
self.id, name, None, feature_group_api.FeatureGroupApi.CACHED
)
for fg_object in feature_group_object:
fg_object.feature_store = self
return feature_group_object
@usage.method_logger
def get_on_demand_feature_group(self, name: str, version: int = None):
"""Get a external feature group entity from the feature store.
!!! warning "Deprecated"
`get_on_demand_feature_group` method is deprecated. Use the `get_external_feature_group` method instead.
Getting a external feature group from the Feature Store means getting its
metadata handle so you can subsequently read the data into a Spark or
Pandas DataFrame or use the `Query`-API to perform joins between feature groups.
# Arguments
name: Name of the external feature group to get.
version: Version of the external feature group to retrieve,
defaults to `None` and will return the `version=1`.
# Returns
`ExternalFeatureGroup`: The external feature group metadata object.
# Raises
`hsfs.client.exceptions.RestAPIError`: If unable to retrieve feature group from the feature store.
"""
return self.get_external_feature_group(name, version)
@usage.method_logger
def get_external_feature_group(self, name: str, version: int = None):
"""Get a external feature group entity from the feature store.
Getting a external feature group from the Feature Store means getting its
metadata handle so you can subsequently read the data into a Spark or
Pandas DataFrame or use the `Query`-API to perform joins between feature groups.
!!! example
```python
# connect to the Feature Store
fs = ...
external_fg = fs.get_external_feature_group("external_fg_test")
```
# Arguments
name: Name of the external feature group to get.
version: Version of the external feature group to retrieve,
defaults to `None` and will return the `version=1`.
# Returns
`ExternalFeatureGroup`: The external feature group metadata object.
# Raises
`hsfs.client.exceptions.RestAPIError`: If unable to retrieve feature group from the feature store.
"""
if version is None:
warnings.warn(
"No version provided for getting feature group `{}`, defaulting to `{}`.".format(
name, self.DEFAULT_VERSION
),
util.VersionWarning,
)
version = self.DEFAULT_VERSION
feature_group_object = self._feature_group_api.get(
self.id, name, version, feature_group_api.FeatureGroupApi.ONDEMAND
)
feature_group_object.feature_store = self
return feature_group_object
@usage.method_logger
def get_on_demand_feature_groups(self, name: str):
"""Get a list of all versions of an external feature group entity from the feature store.
!!! warning "Deprecated"
`get_on_demand_feature_groups` method is deprecated. Use the `get_external_feature_groups` method instead.
Getting a external feature group from the Feature Store means getting its
metadata handle so you can subsequently read the data into a Spark or
Pandas DataFrame or use the `Query`-API to perform joins between feature groups.
# Arguments
name: Name of the external feature group to get.
# Returns
`ExternalFeatureGroup`: List of external feature group metadata objects.
# Raises
`hsfs.client.exceptions.RestAPIError`: If unable to retrieve feature group from the feature store.
"""
return self.get_external_feature_groups(name)
@usage.method_logger
def get_external_feature_groups(self, name: str):
"""Get a list of all versions of an external feature group entity from the feature store.
Getting a external feature group from the Feature Store means getting its
metadata handle so you can subsequently read the data into a Spark or
Pandas DataFrame or use the `Query`-API to perform joins between feature groups.
!!! example
```python
# connect to the Feature Store
fs = ...
external_fgs_list = fs.get_external_feature_groups("external_fg_test")
```
# Arguments
name: Name of the external feature group to get.
# Returns
`ExternalFeatureGroup`: List of external feature group metadata objects.
# Raises
`hsfs.client.exceptions.RestAPIError`: If unable to retrieve feature group from the feature store.
"""
feature_group_object = self._feature_group_api.get(
self.id, name, None, feature_group_api.FeatureGroupApi.ONDEMAND
)
for fg_object in feature_group_object:
fg_object.feature_store = self
return feature_group_object
def get_training_dataset(self, name: str, version: int = None):
"""Get a training dataset entity from the feature store.
!!! warning "Deprecated"
`TrainingDataset` is deprecated, use `FeatureView` instead. You can still retrieve old
training datasets using this method, but after upgrading the old training datasets will
also be available under a Feature View with the same name and version.
It is recommended to use this method only for old training datasets that have been
created directly from Dataframes and not with Query objects.
Getting a training dataset from the Feature Store means getting its metadata handle
so you can subsequently read the data into a Spark or Pandas DataFrame.
# Arguments
name: Name of the training dataset to get.
version: Version of the training dataset to retrieve, defaults to `None` and will
return the `version=1`.
# Returns
`TrainingDataset`: The training dataset metadata object.
# Raises
`hsfs.client.exceptions.RestAPIError`: If unable to retrieve feature group from the feature store.
"""
if version is None:
warnings.warn(
"No version provided for getting training dataset `{}`, defaulting to `{}`.".format(
name, self.DEFAULT_VERSION
),
util.VersionWarning,
)
version = self.DEFAULT_VERSION
return self._training_dataset_api.get(name, version)
def get_training_datasets(self, name: str):
"""Get a list of all versions of a training dataset entity from the feature store.
!!! warning "Deprecated"
`TrainingDataset` is deprecated, use `FeatureView` instead.
Getting a training dataset from the Feature Store means getting its metadata handle
so you can subsequently read the data into a Spark or Pandas DataFrame.
# Arguments
name: Name of the training dataset to get.
# Returns
`TrainingDataset`: List of training dataset metadata objects.
# Raises
`hsfs.client.exceptions.RestAPIError`: If unable to retrieve feature group from the feature store.
"""
return self._training_dataset_api.get(name, None)
@usage.method_logger
def get_storage_connector(self, name: str):
"""Get a previously created storage connector from the feature store.
Storage connectors encapsulate all information needed for the execution engine
to read and write to specific storage. This storage can be S3, a JDBC compliant
database or the distributed filesystem HOPSFS.
If you want to connect to the online feature store, see the
`get_online_storage_connector` method to get the JDBC connector for the Online
Feature Store.
!!! example
```python
# connect to the Feature Store
fs = ...
sc = fs.get_storage_connector("demo_fs_meb10000_Training_Datasets")
```
# Arguments
name: Name of the storage connector to retrieve.
# Returns
`StorageConnector`. Storage connector object.
"""
return self._storage_connector_api.get(self._id, name)
def sql(
self,
query: str,
dataframe_type: Optional[str] = "default",
online: Optional[bool] = False,
read_options: Optional[dict] = {},
):
"""Execute SQL command on the offline or online feature store database
!!! example
```python
# connect to the Feature Store
fs = ...
# construct the query and show head rows
query_res_head = fs.sql(\"SELECT * FROM `fg_1`\").head()
```
# Arguments
query: The SQL query to execute.
dataframe_type: The type of the returned dataframe. Defaults to "default".
which maps to Spark dataframe for the Spark Engine and Pandas dataframe for the Hive engine.
online: Set to true to execute the query against the online feature store.
Defaults to False.
read_options: Additional options as key/value pairs to pass to the execution engine.
For spark engine: Dictionary of read options for Spark.
For python engine:
* key `"hive_config"` to pass a dictionary of hive or tez configurations.
For example: `{"hive_config": {"hive.tez.cpu.vcores": 2, "tez.grouping.split-count": "3"}}`
If running queries on the online feature store, users can provide an entry `{'external': True}`,
this instructs the library to use the `host` parameter in the [`hsfs.connection()`](connection_api.md#connection) to establish the connection to the online feature store.
If not set, or set to False, the online feature store storage connector is used which relies on
the private ip.
Defaults to `{}`.
# Returns
`DataFrame`: DataFrame depending on the chosen type.
"""
return self._feature_group_engine.sql(
query, self._name, dataframe_type, online, read_options
)
@usage.method_logger
def get_online_storage_connector(self):
"""Get the storage connector for the Online Feature Store of the respective
project's feature store.
The returned storage connector depends on the project that you are connected to.
!!! example
```python
# connect to the Feature Store
fs = ...
online_storage_connector = fs.get_online_storage_connector()
```
# Returns
`StorageConnector`. JDBC storage connector to the Online Feature Store.
"""
return self._storage_connector_api.get_online_connector(self._id)
@usage.method_logger
def create_feature_group(
self,
name: str,
version: Optional[int] = None,
description: Optional[str] = "",
online_enabled: Optional[bool] = False,
time_travel_format: Optional[str] = "HUDI",
partition_key: Optional[List[str]] = [],
primary_key: Optional[List[str]] = [],
hudi_precombine_key: Optional[str] = None,
features: Optional[List[feature.Feature]] = [],
statistics_config: Optional[Union[StatisticsConfig, bool, dict]] = None,
event_time: Optional[str] = None,
stream: Optional[bool] = False,
expectation_suite: Optional[
Union[expectation_suite.ExpectationSuite, ge.core.ExpectationSuite]
] = None,
parents: Optional[List[feature_group.FeatureGroup]] = [],
topic_name: Optional[str] = None,
):
"""Create a feature group metadata object.
!!! example
```python
# connect to the Feature Store
fs = ...
fg = fs.create_feature_group(
name='air_quality',
description='Air Quality characteristics of each day',
version=1,
primary_key=['city','date'],
online_enabled=True,
event_time='date'
)
```
!!! note "Lazy"
This method is lazy and does not persist any metadata or feature data in the
feature store on its own. To persist the feature group and save feature data
along the metadata in the feature store, call the `save()` method with a
DataFrame.
# Arguments
name: Name of the feature group to create.
version: Version of the feature group to retrieve, defaults to `None` and
will create the feature group with incremented version from the last
version in the feature store.
description: A string describing the contents of the feature group to
improve discoverability for Data Scientists, defaults to empty string
`""`.
online_enabled: Define whether the feature group should be made available
also in the online feature store for low latency access, defaults to
`False`.
time_travel_format: Format used for time travel, defaults to `"HUDI"`.
partition_key: A list of feature names to be used as partition key when
writing the feature data to the offline storage, defaults to empty list
`[]`.
primary_key: A list of feature names to be used as primary key for the
feature group. This primary key can be a composite key of multiple
features and will be used as joining key, if not specified otherwise.
Defaults to empty list `[]`, and the feature group won't have any primary key.
hudi_precombine_key: A feature name to be used as a precombine key for the `"HUDI"`
feature group. Defaults to `None`. If feature group has time travel format
`"HUDI"` and hudi precombine key was not specified then the first primary key of
the feature group will be used as hudi precombine key.
features: Optionally, define the schema of the feature group manually as a
list of `Feature` objects. Defaults to empty list `[]` and will use the
schema information of the DataFrame provided in the `save` method.
statistics_config: A configuration object, or a dictionary with keys
"`enabled`" to generally enable descriptive statistics computation for
this feature group, `"correlations`" to turn on feature correlation
computation, `"histograms"` to compute feature value frequencies and
`"exact_uniqueness"` to compute uniqueness, distinctness and entropy.
The values should be booleans indicating the setting. To fully turn off
statistics computation pass `statistics_config=False`. Defaults to
`None` and will compute only descriptive statistics.
event_time: Optionally, provide the name of the feature containing the event
time for the features in this feature group. If event_time is set
the feature group can be used for point-in-time joins. Defaults to `None`.
!!!note "Event time data type restriction"
The supported data types for the event time column are: `timestamp`, `date` and `bigint`.
stream: Optionally, Define whether the feature group should support real time stream writing capabilities.
Stream enabled Feature Groups have unified single API for writing streaming features transparently
to both online and offline store.
expectation_suite: Optionally, attach an expectation suite to the feature
group which dataframes should be validated against upon insertion.
Defaults to `None`.
parents: Optionally, Define the parents of this feature group as the
origin where the data is coming from.
topic_name: Optionally, define the name of the topic used for data ingestion. If left undefined it
defaults to using project topic.
# Returns
`FeatureGroup`. The feature group metadata object.
"""
feature_group_object = feature_group.FeatureGroup(
name=name,
version=version,
description=description,
online_enabled=online_enabled,
time_travel_format=time_travel_format,
partition_key=partition_key,
primary_key=primary_key,
hudi_precombine_key=hudi_precombine_key,
featurestore_id=self._id,
featurestore_name=self._name,
features=features,
statistics_config=statistics_config,
event_time=event_time,
stream=stream,
expectation_suite=expectation_suite,
parents=parents,
topic_name=topic_name,
)
feature_group_object.feature_store = self
return feature_group_object
@usage.method_logger
def get_or_create_feature_group(
self,
name: str,
version: int,
description: Optional[str] = "",
online_enabled: Optional[bool] = False,
time_travel_format: Optional[str] = "HUDI",
partition_key: Optional[List[str]] = [],
primary_key: Optional[List[str]] = [],
hudi_precombine_key: Optional[str] = None,
features: Optional[List[feature.Feature]] = [],
statistics_config: Optional[Union[StatisticsConfig, bool, dict]] = None,
expectation_suite: Optional[
Union[expectation_suite.ExpectationSuite, ge.core.ExpectationSuite]
] = None,
event_time: Optional[str] = None,
stream: Optional[bool] = False,
parents: Optional[List[feature_group.FeatureGroup]] = [],
topic_name: Optional[str] = None,
):
"""Get feature group metadata object or create a new one if it doesn't exist. This method doesn't update existing feature group metadata object.
!!! example
```python
# connect to the Feature Store
fs = ...
fg = fs.get_or_create_feature_group(
name="electricity_prices",
version=1,
description="Electricity prices from NORD POOL",
primary_key=["day", "area"],
online_enabled=True,
event_time="timestamp",
)
```
!!! note "Lazy"
This method is lazy and does not persist any metadata or feature data in the
feature store on its own. To persist the feature group and save feature data
along the metadata in the feature store, call the `insert()` method with a
DataFrame.
# Arguments
name: Name of the feature group to create.
version: Version of the feature group to retrieve or create.
description: A string describing the contents of the feature group to
improve discoverability for Data Scientists, defaults to empty string
`""`.
online_enabled: Define whether the feature group should be made available
also in the online feature store for low latency access, defaults to
`False`.
time_travel_format: Format used for time travel, defaults to `"HUDI"`.
partition_key: A list of feature names to be used as partition key when
writing the feature data to the offline storage, defaults to empty list
`[]`.
primary_key: A list of feature names to be used as primary key for the
feature group. This primary key can be a composite key of multiple
features and will be used as joining key, if not specified otherwise.
Defaults to empty list `[]`, and the feature group won't have any primary key.
hudi_precombine_key: A feature name to be used as a precombine key for the `"HUDI"`
feature group. Defaults to `None`. If feature group has time travel format
`"HUDI"` and hudi precombine key was not specified then the first primary key of
the feature group will be used as hudi precombine key.
features: Optionally, define the schema of the feature group manually as a
list of `Feature` objects. Defaults to empty list `[]` and will use the
schema information of the DataFrame provided in the `save` method.
statistics_config: A configuration object, or a dictionary with keys
"`enabled`" to generally enable descriptive statistics computation for
this feature group, `"correlations`" to turn on feature correlation
computation, `"histograms"` to compute feature value frequencies and
`"exact_uniqueness"` to compute uniqueness, distinctness and entropy.
The values should be booleans indicating the setting. To fully turn off
statistics computation pass `statistics_config=False`. Defaults to
`None` and will compute only descriptive statistics.
expectation_suite: Optionally, attach an expectation suite to the feature
group which dataframes should be validated against upon insertion.
Defaults to `None`.
event_time: Optionally, provide the name of the feature containing the event
time for the features in this feature group. If event_time is set
the feature group can be used for point-in-time joins. Defaults to `None`.
!!!note "Event time data type restriction"
The supported data types for the event time column are: `timestamp`, `date` and `bigint`.
stream: Optionally, Define whether the feature group should support real time stream writing capabilities.
Stream enabled Feature Groups have unified single API for writing streaming features transparently
to both online and offline store.
parents: Optionally, Define the parents of this feature group as the
origin where the data is coming from.
topic_name: Optionally, define the name of the topic used for data ingestion. If left undefined it
defaults to using project topic.
# Returns
`FeatureGroup`. The feature group metadata object.
"""
try:
feature_group_object = self._feature_group_api.get(
self.id, name, version, feature_group_api.FeatureGroupApi.CACHED
)
feature_group_object.feature_store = self
return feature_group_object
except exceptions.RestAPIError as e:
if (
e.response.json().get("errorCode", "") == 270009
and e.response.status_code == 404
):
feature_group_object = feature_group.FeatureGroup(
name=name,
version=version,
description=description,
online_enabled=online_enabled,
time_travel_format=time_travel_format,
partition_key=partition_key,
primary_key=primary_key,
hudi_precombine_key=hudi_precombine_key,
featurestore_id=self._id,
featurestore_name=self._name,
features=features,
statistics_config=statistics_config,
event_time=event_time,
stream=stream,
expectation_suite=expectation_suite,
parents=parents,
topic_name=topic_name,
)
feature_group_object.feature_store = self
return feature_group_object
else:
raise e
@usage.method_logger
def create_on_demand_feature_group(
self,
name: str,
storage_connector: storage_connector.StorageConnector,
query: Optional[str] = None,
data_format: Optional[str] = None,
path: Optional[str] = "",
options: Optional[Dict[str, str]] = {},
version: Optional[int] = None,
description: Optional[str] = "",
primary_key: Optional[List[str]] = [],
features: Optional[List[feature.Feature]] = [],
statistics_config: Optional[Union[StatisticsConfig, bool, dict]] = None,
event_time: Optional[str] = None,
expectation_suite: Optional[
Union[expectation_suite.ExpectationSuite, ge.core.ExpectationSuite]
] = None,
topic_name: Optional[str] = None,
):
"""Create a external feature group metadata object.
!!! warning "Deprecated"
`create_on_demand_feature_group` method is deprecated. Use the `create_external_feature_group` method instead.
!!! note "Lazy"
This method is lazy and does not persist any metadata in the
feature store on its own. To persist the feature group metadata in the feature store,
call the `save()` method.
# Arguments
name: Name of the external feature group to create.
storage_connector: the storage connector to use to establish connectivity
with the data source.
query: A string containing a SQL query valid for the target data source.
the query will be used to pull data from the data sources when the
feature group is used.
data_format: If the external feature groups refers to a directory with data,
the data format to use when reading it
path: The location within the scope of the storage connector, from where to read
the data for the external feature group
options: Additional options to be used by the engine when reading data from the
specified storage connector. For example, `{"header": True}` when reading
CSV files with column names in the first row.
version: Version of the external feature group to retrieve, defaults to `None` and
will create the feature group with incremented version from the last
version in the feature store.
description: A string describing the contents of the external feature group to
improve discoverability for Data Scientists, defaults to empty string
`""`.
primary_key: A list of feature names to be used as primary key for the
feature group. This primary key can be a composite key of multiple
features and will be used as joining key, if not specified otherwise.
Defaults to empty list `[]`, and the feature group won't have any primary key.
features: Optionally, define the schema of the external feature group manually as a
list of `Feature` objects. Defaults to empty list `[]` and will use the
schema information of the DataFrame resulting by executing the provided query
against the data source.
statistics_config: A configuration object, or a dictionary with keys
"`enabled`" to generally enable descriptive statistics computation for
this external feature group, `"correlations`" to turn on feature correlation
computation, `"histograms"` to compute feature value frequencies and
`"exact_uniqueness"` to compute uniqueness, distinctness and entropy.
The values should be booleans indicating the setting. To fully turn off
statistics computation pass `statistics_config=False`. Defaults to
`None` and will compute only descriptive statistics.
event_time: Optionally, provide the name of the feature containing the event
time for the features in this feature group. If event_time is set
the feature group can be used for point-in-time joins. Defaults to `None`.
topic_name: Optionally, define the name of the topic used for data ingestion. If left undefined it
defaults to using project topic.
!!!note "Event time data type restriction"
The supported data types for the event time column are: `timestamp`, `date` and `bigint`.
expectation_suite: Optionally, attach an expectation suite to the feature
group which dataframes should be validated against upon insertion.
Defaults to `None`.
# Returns
`ExternalFeatureGroup`. The external feature group metadata object.
"""
feature_group_object = feature_group.ExternalFeatureGroup(
name=name,
query=query,
data_format=data_format,
path=path,
options=options,
storage_connector=storage_connector,
version=version,
description=description,
primary_key=primary_key,
featurestore_id=self._id,
featurestore_name=self._name,
features=features,
statistics_config=statistics_config,
event_time=event_time,
expectation_suite=expectation_suite,
topic_name=topic_name,
)
feature_group_object.feature_store = self
return feature_group_object
@usage.method_logger
def create_external_feature_group(
self,
name: str,
storage_connector: storage_connector.StorageConnector,
query: Optional[str] = None,
data_format: Optional[str] = None,
path: Optional[str] = "",
options: Optional[Dict[str, str]] = {},
version: Optional[int] = None,
description: Optional[str] = "",
primary_key: Optional[List[str]] = [],
features: Optional[List[feature.Feature]] = [],
statistics_config: Optional[Union[StatisticsConfig, bool, dict]] = None,
event_time: Optional[str] = None,
expectation_suite: Optional[
Union[expectation_suite.ExpectationSuite, ge.core.ExpectationSuite]
] = None,
online_enabled: Optional[bool] = False,
topic_name: Optional[str] = None,
):
"""Create a external feature group metadata object.
!!! example
```python
# connect to the Feature Store
fs = ...
external_fg = fs.create_external_feature_group(
name="sales",
version=1,
description="Physical shop sales features",
query=query,
storage_connector=connector,
primary_key=['ss_store_sk'],
event_time='sale_date'
)
```
!!! note "Lazy"
This method is lazy and does not persist any metadata in the
feature store on its own. To persist the feature group metadata in the feature store,
call the `save()` method.
You can enable online storage for external feature groups, however, the sync from the
external storage to Hopsworks online storage needs to be done manually:
```python
external_fg = fs.create_external_feature_group(
name="sales",
version=1,
description="Physical shop sales features",
query=query,
storage_connector=connector,
primary_key=['ss_store_sk'],
event_time='sale_date',
online_enabled=True
)
external_fg.save()
# read from external storage and filter data to sync to online
df = external_fg.read().filter(external_fg.customer_status == "active")
# insert to online storage
external_fg.insert(df)
```
# Arguments
name: Name of the external feature group to create.
storage_connector: the storage connector to use to establish connectivity
with the data source.
query: A string containing a SQL query valid for the target data source.
the query will be used to pull data from the data sources when the
feature group is used.
data_format: If the external feature groups refers to a directory with data,
the data format to use when reading it
path: The location within the scope of the storage connector, from where to read
the data for the external feature group
options: Additional options to be used by the engine when reading data from the
specified storage connector. For example, `{"header": True}` when reading
CSV files with column names in the first row.
version: Version of the external feature group to retrieve, defaults to `None` and
will create the feature group with incremented version from the last
version in the feature store.
description: A string describing the contents of the external feature group to
improve discoverability for Data Scientists, defaults to empty string
`""`.
primary_key: A list of feature names to be used as primary key for the
feature group. This primary key can be a composite key of multiple
features and will be used as joining key, if not specified otherwise.
Defaults to empty list `[]`, and the feature group won't have any primary key.
features: Optionally, define the schema of the external feature group manually as a
list of `Feature` objects. Defaults to empty list `[]` and will use the
schema information of the DataFrame resulting by executing the provided query
against the data source.
statistics_config: A configuration object, or a dictionary with keys
"`enabled`" to generally enable descriptive statistics computation for
this external feature group, `"correlations`" to turn on feature correlation
computation, `"histograms"` to compute feature value frequencies and
`"exact_uniqueness"` to compute uniqueness, distinctness and entropy.
The values should be booleans indicating the setting. To fully turn off
statistics computation pass `statistics_config=False`. Defaults to
`None` and will compute only descriptive statistics.
event_time: Optionally, provide the name of the feature containing the event
time for the features in this feature group. If event_time is set
the feature group can be used for point-in-time joins. Defaults to `None`.
!!! note "Event time data type restriction"
The supported data types for the event time column are: `timestamp`, `date` and `bigint`.
expectation_suite: Optionally, attach an expectation suite to the feature
group which dataframes should be validated against upon insertion.
Defaults to `None`.
online_enabled: Define whether it should be possible to sync the feature group to
the online feature store for low latency access, defaults to `False`.
topic_name: Optionally, define the name of the topic used for data ingestion. If left undefined it
defaults to using project topic.
# Returns
`ExternalFeatureGroup`. The external feature group metadata object.
"""
feature_group_object = feature_group.ExternalFeatureGroup(
name=name,
query=query,
data_format=data_format,
path=path,
options=options,
storage_connector=storage_connector,
version=version,
description=description,
primary_key=primary_key,
featurestore_id=self._id,
featurestore_name=self._name,
features=features,
statistics_config=statistics_config,
event_time=event_time,
expectation_suite=expectation_suite,
online_enabled=online_enabled,
topic_name=topic_name,
)
feature_group_object.feature_store = self
return feature_group_object
@usage.method_logger
def get_or_create_spine_group(
self,
name: str,
version: Optional[int] = None,
description: Optional[str] = "",
primary_key: Optional[List[str]] = [],
event_time: Optional[str] = None,
features: Optional[List[feature.Feature]] = [],
dataframe: Union[
pd.DataFrame,
TypeVar("pyspark.sql.DataFrame"), # noqa: F821
TypeVar("pyspark.RDD"), # noqa: F821
np.ndarray,
List[list],
] = None,
):
"""Create a spine group metadata object.
Instead of using a feature group to save a label/prediction target, you can use a spine together with a dataframe containing the labels and join keys for features in other feature groups.
A Spine is essentially a metadata object similar to a feature group, however, its data is not stored in the feature store.
The Spine stored in the feature store only contains the needed metadata such as the name, version, primary key column(s), and event time column.
The Spine DataFrame is provided when you need to (1) create training data and (2) create batch inference data. The Spine DataFrame should also contain any join keys (primary keys to other feature groups) needed to join features included in a feature view containing the Spine group.If you don’t include the event_time in the Spine DataFrame (such as in batch inference), it will retrieve the latest feature value for that feature using the join key(s).
The main uses of a Spine Group in Hopsworks are:
1. in model training to enable users to provide labels as a DataFrame,
2. in batch inference to retrieve feature values using an event_time and primary key provided by the Spine DataFrame.
!!! example
```python
# connect to the Feature Store
fs = ...
spine_df = pd.Dataframe()
spine_group = fs.get_or_create_spine_group(
name="sales",
version=1,
description="Physical shop sales features",
primary_key=['ss_store_sk'],
event_time='sale_date',
dataframe=spine_df
)
```
Note that you can inspect the DataFrame in the spine group, or replace the DataFrame:
```python
spine_group.dataframe.show()
spine_group.dataframe = new_df