arctix.dataset
arctix.dataset.breakfast ¶
Contain code to download and prepare the Breakfast data.
The following documentation assumes the data are downloaded in the
directory /path/to/data/breakfast/.
arctix.dataset.breakfast.fetch_data ¶
fetch_data(
path: Path,
name: str,
remove_duplicate: bool = True,
force_download: bool = False,
) -> DataFrame
Download and load the data for Breakfast dataset.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
path
|
Path
|
The path where to store the downloaded data. |
required |
name
|
str
|
The name of the dataset. The valid names are
|
required |
remove_duplicate
|
bool
|
If |
True
|
force_download
|
bool
|
If |
False
|
Returns:
| Type | Description |
|---|---|
DataFrame
|
The data in a DataFrame |
Raises:
| Type | Description |
|---|---|
RuntimeError
|
if the name is incorrect |
Example usage:
>>> from pathlib import Path
>>> from arctix.dataset.breakfast import fetch_data
>>> data = fetch_data(
... Path("/path/to/data/breakfast/"), "segmentation_coarse"
... ) # doctest: +SKIP
arctix.dataset.breakfast.prepare_data ¶
prepare_data(
frame: DataFrame, split: str = "all"
) -> tuple[DataFrame, dict]
Prepare the data.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
frame
|
DataFrame
|
The raw DataFrame. |
required |
split
|
str
|
The dataset split. By default, the union of all the dataset splits is used. |
'all'
|
Returns:
| Type | Description |
|---|---|
tuple[DataFrame, dict]
|
A tuple containing the prepared data and the metadata. |
Example usage:
>>> import polars as pl
>>> from arctix.dataset.breakfast import Column, group_by_sequence
>>> frame = pl.DataFrame(
... {
... Column.ACTION: [
... "SIL",
... "take_bowl",
... "pour_cereals",
... "pour_milk",
... "stir_cereals",
... "SIL",
... "SIL",
... "pour_milk",
... "spoon_powder",
... "SIL",
... ],
... Column.COOKING_ACTIVITY: [
... "cereals",
... "cereals",
... "cereals",
... "cereals",
... "cereals",
... "cereals",
... "milk",
... "milk",
... "milk",
... "milk",
... ],
... Column.END_TIME: [
... 30.0,
... 150.0,
... 428.0,
... 575.0,
... 705.0,
... 836.0,
... 47.0,
... 215.0,
... 565.0,
... 747.0,
... ],
... Column.PERSON: [
... "P03",
... "P03",
... "P03",
... "P03",
... "P03",
... "P03",
... "P54",
... "P54",
... "P54",
... "P54",
... ],
... Column.START_TIME: [
... 1.0,
... 31.0,
... 151.0,
... 429.0,
... 576.0,
... 706.0,
... 1.0,
... 48.0,
... 216.0,
... 566.0,
... ],
... },
... )
>>> data, metadata = prepare_data(frame)
>>> with pl.Config(tbl_cols=-1):
... data
...
shape: (10, 9)
┌───────────┬───────────┬──────────┬──────────┬──────────┬────────┬──────────┬──────────┬──────────┐
│ action ┆ action_id ┆ cooking_ ┆ cooking_ ┆ end_time ┆ person ┆ person_i ┆ start_ti ┆ start_ti │
│ --- ┆ --- ┆ activity ┆ activity ┆ --- ┆ --- ┆ d ┆ me ┆ me_diff │
│ str ┆ i64 ┆ --- ┆ _id ┆ f64 ┆ str ┆ --- ┆ --- ┆ --- │
│ ┆ ┆ str ┆ --- ┆ ┆ ┆ i64 ┆ f64 ┆ f64 │
│ ┆ ┆ ┆ i64 ┆ ┆ ┆ ┆ ┆ │
╞═══════════╪═══════════╪══════════╪══════════╪══════════╪════════╪══════════╪══════════╪══════════╡
│ SIL ┆ 0 ┆ cereals ┆ 0 ┆ 30.0 ┆ P03 ┆ 0 ┆ 1.0 ┆ 0.0 │
│ take_bowl ┆ 2 ┆ cereals ┆ 0 ┆ 150.0 ┆ P03 ┆ 0 ┆ 31.0 ┆ 30.0 │
│ pour_cere ┆ 5 ┆ cereals ┆ 0 ┆ 428.0 ┆ P03 ┆ 0 ┆ 151.0 ┆ 120.0 │
│ als ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ │
│ pour_milk ┆ 1 ┆ cereals ┆ 0 ┆ 575.0 ┆ P03 ┆ 0 ┆ 429.0 ┆ 278.0 │
│ stir_cere ┆ 3 ┆ cereals ┆ 0 ┆ 705.0 ┆ P03 ┆ 0 ┆ 576.0 ┆ 147.0 │
│ als ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ │
│ SIL ┆ 0 ┆ cereals ┆ 0 ┆ 836.0 ┆ P03 ┆ 0 ┆ 706.0 ┆ 130.0 │
│ SIL ┆ 0 ┆ milk ┆ 1 ┆ 47.0 ┆ P54 ┆ 1 ┆ 1.0 ┆ 0.0 │
│ pour_milk ┆ 1 ┆ milk ┆ 1 ┆ 215.0 ┆ P54 ┆ 1 ┆ 48.0 ┆ 47.0 │
│ spoon_pow ┆ 4 ┆ milk ┆ 1 ┆ 565.0 ┆ P54 ┆ 1 ┆ 216.0 ┆ 168.0 │
│ der ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ │
│ SIL ┆ 0 ┆ milk ┆ 1 ┆ 747.0 ┆ P54 ┆ 1 ┆ 566.0 ┆ 350.0 │
└───────────┴───────────┴──────────┴──────────┴──────────┴────────┴──────────┴──────────┴──────────┘
>>> metadata
{'vocab_action': Vocabulary(
counter=Counter({'SIL': 4, 'pour_milk': 2, 'take_bowl': 1, 'stir_cereals': 1, 'spoon_powder': 1, 'pour_cereals': 1}),
index_to_token=('SIL', 'pour_milk', 'take_bowl', 'stir_cereals', 'spoon_powder', 'pour_cereals'),
token_to_index={'SIL': 0, 'pour_milk': 1, 'take_bowl': 2, 'stir_cereals': 3, 'spoon_powder': 4, 'pour_cereals': 5},
), 'vocab_activity': Vocabulary(
counter=Counter({'cereals': 6, 'milk': 4}),
index_to_token=('cereals', 'milk'),
token_to_index={'cereals': 0, 'milk': 1},
), 'vocab_person': Vocabulary(
counter=Counter({'P03': 6, 'P54': 4}),
index_to_token=('P03', 'P54'),
token_to_index={'P03': 0, 'P54': 1},
)}
arctix.dataset.breakfast.to_array ¶
to_array(frame: DataFrame) -> dict[str, ndarray]
Convert a DataFrame to a dictionary of arrays.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
frame
|
DataFrame
|
The input DataFrame. |
required |
Returns:
| Type | Description |
|---|---|
dict[str, ndarray]
|
The dictionary of arrays. |
Example usage:
>>> import polars as pl
>>> from arctix.dataset.breakfast import Column, to_array
>>> frame = pl.DataFrame(
... {
... Column.ACTION: [
... "SIL",
... "take_bowl",
... "pour_cereals",
... "pour_milk",
... "stir_cereals",
... "SIL",
... "SIL",
... "pour_milk",
... "spoon_powder",
... "SIL",
... ],
... Column.ACTION_ID: [0, 2, 5, 1, 3, 0, 0, 1, 4, 0],
... Column.COOKING_ACTIVITY: [
... "cereals",
... "cereals",
... "cereals",
... "cereals",
... "cereals",
... "cereals",
... "milk",
... "milk",
... "milk",
... "milk",
... ],
... Column.COOKING_ACTIVITY_ID: [0, 0, 0, 0, 0, 0, 1, 1, 1, 1],
... Column.END_TIME: [
... 30.0,
... 150.0,
... 428.0,
... 575.0,
... 705.0,
... 836.0,
... 47.0,
... 215.0,
... 565.0,
... 747.0,
... ],
... Column.PERSON: [
... "P03",
... "P03",
... "P03",
... "P03",
... "P03",
... "P03",
... "P54",
... "P54",
... "P54",
... "P54",
... ],
... Column.PERSON_ID: [0, 0, 0, 0, 0, 0, 1, 1, 1, 1],
... Column.START_TIME: [
... 1.0,
... 31.0,
... 151.0,
... 429.0,
... 576.0,
... 706.0,
... 1.0,
... 48.0,
... 216.0,
... 566.0,
... ],
... Column.START_TIME_DIFF: [
... 0.0,
... 30.0,
... 120.0,
... 278.0,
... 147.0,
... 130.0,
... 0.0,
... 47.0,
... 168.0,
... 350.0,
... ],
... }
... )
>>> arrays = to_array(frame)
>>> arrays
{'action': masked_array(
data=[['SIL', 'take_bowl', 'pour_cereals', 'pour_milk', 'stir_cereals',
'SIL'],
['SIL', 'pour_milk', 'spoon_powder', 'SIL', --, --]],
mask=[[False, False, False, False, False, False],
[False, False, False, False, True, True]],
fill_value='N/A',
dtype='<U12'),
'action_id': masked_array(
data=[[0, 2, 5, 1, 3, 0],
[0, 1, 4, 0, --, --]],
mask=[[False, False, False, False, False, False],
[False, False, False, False, True, True]],
fill_value=999999),
'cooking_activity': array(['cereals', 'milk'], dtype='<U7'),
'cooking_activity_id': array([0, 1]), 'person': array(['P03', 'P54'], dtype='<U3'),
'person_id': array([0, 1]),
'sequence_length': array([6, 4]),
'start_time': masked_array(
data=[[1.0, 31.0, 151.0, 429.0, 576.0, 706.0],
[1.0, 48.0, 216.0, 566.0, --, --]],
mask=[[False, False, False, False, False, False],
[False, False, False, False, True, True]],
fill_value=1e+20), 'start_time_diff': masked_array(
data=[[0.0, 30.0, 120.0, 278.0, 147.0, 130.0],
[0.0, 47.0, 168.0, 350.0, --, --]],
mask=[[False, False, False, False, False, False],
[False, False, False, False, True, True]],
fill_value=1e+20), 'end_time': masked_array(
data=[[30.0, 150.0, 428.0, 575.0, 705.0, 836.0],
[47.0, 215.0, 565.0, 747.0, --, --]],
mask=[[False, False, False, False, False, False],
[False, False, False, False, True, True]],
fill_value=1e+20)}
arctix.dataset.breakfast.to_list ¶
to_list(frame: DataFrame) -> dict[str, list]
Convert a DataFrame to a dictionary of lists.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
frame
|
DataFrame
|
The input DataFrame. |
required |
Returns:
| Type | Description |
|---|---|
dict[str, list]
|
The dictionary of lists. |
Example usage:
>>> import polars as pl
>>> from arctix.dataset.breakfast import Column, to_list
>>> frame = pl.DataFrame(
... {
... Column.ACTION: [
... "SIL",
... "take_bowl",
... "pour_cereals",
... "pour_milk",
... "stir_cereals",
... "SIL",
... "SIL",
... "pour_milk",
... "spoon_powder",
... "SIL",
... ],
... Column.ACTION_ID: [0, 2, 5, 1, 3, 0, 0, 1, 4, 0],
... Column.COOKING_ACTIVITY: [
... "cereals",
... "cereals",
... "cereals",
... "cereals",
... "cereals",
... "cereals",
... "milk",
... "milk",
... "milk",
... "milk",
... ],
... Column.COOKING_ACTIVITY_ID: [0, 0, 0, 0, 0, 0, 1, 1, 1, 1],
... Column.END_TIME: [
... 30.0,
... 150.0,
... 428.0,
... 575.0,
... 705.0,
... 836.0,
... 47.0,
... 215.0,
... 565.0,
... 747.0,
... ],
... Column.PERSON: [
... "P03",
... "P03",
... "P03",
... "P03",
... "P03",
... "P03",
... "P54",
... "P54",
... "P54",
... "P54",
... ],
... Column.PERSON_ID: [0, 0, 0, 0, 0, 0, 1, 1, 1, 1],
... Column.START_TIME: [
... 1.0,
... 31.0,
... 151.0,
... 429.0,
... 576.0,
... 706.0,
... 1.0,
... 48.0,
... 216.0,
... 566.0,
... ],
... Column.START_TIME_DIFF: [
... 0.0,
... 30.0,
... 120.0,
... 278.0,
... 147.0,
... 130.0,
... 0.0,
... 47.0,
... 168.0,
... 350.0,
... ],
... }
... )
>>> data_list = to_list(frame)
>>> data_list
{'action': [['SIL', 'take_bowl', 'pour_cereals', 'pour_milk', 'stir_cereals', 'SIL'], ['SIL', 'pour_milk', 'spoon_powder', 'SIL']],
'action_id': [[0, 2, 5, 1, 3, 0], [0, 1, 4, 0]],
'cooking_activity': ['cereals', 'milk'],
'cooking_activity_id': [0, 1],
'end_time': [[30.0, 150.0, 428.0, 575.0, 705.0, 836.0], [47.0, 215.0, 565.0, 747.0]],
'person': ['P03', 'P54'],
'person_id': [0, 1],
'sequence_length': [6, 4],
'start_time': [[1.0, 31.0, 151.0, 429.0, 576.0, 706.0], [1.0, 48.0, 216.0, 566.0]],
'start_time_diff': [[0.0, 30.0, 120.0, 278.0, 147.0, 130.0], [0.0, 47.0, 168.0, 350.0]]}
arctix.dataset.ego4d ¶
Contain code to prepare the Ego4D data.
The following documentation assumes the data are downloaded in the
directory /path/to/data/ego4d/.
arctix.dataset.ego4d.fetch_data ¶
fetch_data(
path: Path, split: str
) -> tuple[DataFrame, dict]
Download and load the data and the metadata.
Notes
This function does not implement the data downloading because it is necessary to get credentials to access the data.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
path
|
Path
|
The directory where the dataset annotations are stored. |
required |
split
|
str
|
The dataset split. |
required |
Returns:
| Type | Description |
|---|---|
tuple[DataFrame, dict]
|
The annotations in a DataFrame and the metadata. |
Example usage:
>>> from pathlib import Path
>>> from arctix.dataset.ego4d import fetch_data
>>> data, metadata = fetch_data(
... Path("/path/to/data/ego4d/"), split="train"
... ) # doctest: +SKIP
arctix.dataset.ego4d.prepare_data ¶
prepare_data(
frame: DataFrame,
metadata: dict,
group_col: str = CLIP_ID,
) -> tuple[DataFrame, dict]
Prepare the data.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
frame
|
DataFrame
|
The raw DataFrame. |
required |
metadata
|
dict
|
The metadata wich contains the vocabularies to convert verbs and nouns to index. |
required |
group_col
|
str
|
The column used to generate the sequences. |
CLIP_ID
|
Returns:
| Type | Description |
|---|---|
tuple[DataFrame, dict]
|
A tuple containing the prepared data and the metadata. |
Example usage:
>>> import polars as pl
>>> from arctix.dataset.ego4d import Column, prepare_data
>>> frame = pl.DataFrame(
... {
... Column.ACTION_END_FRAME: [47, 82, 102, 74, 142],
... Column.ACTION_END_SEC: [4.7, 8.2, 10.2, 7.4, 14.2],
... Column.ACTION_START_FRAME: [23, 39, 74, 12, 82],
... Column.ACTION_START_SEC: [2.3, 3.9, 7.4, 1.2, 8.2],
... Column.ACTION_INDEX: [0, 1, 2, 0, 1],
... Column.CLIP_ID: ["clip1", "clip1", "clip1", "clip2", "clip2"],
... Column.NOUN: ["noun2", "noun3", "noun1", "noun1", "noun2"],
... Column.NOUN_ID: [2, 3, 1, 1, 2],
... Column.SPLIT: ["train", "train", "train", "train", "train"],
... Column.VERB: ["verb4", "verb2", "verb1", "verb1", "verb2"],
... Column.VERB_ID: [4, 2, 1, 1, 2],
... Column.VIDEO_ID: ["video1", "video1", "video1", "video2", "video2"],
... }
... )
>>> data, metadata = prepare_data(frame, metadata={})
>>> with pl.Config(tbl_cols=-1):
... data
...
shape: (5, 13)
┌─────┬─────┬─────┬────────┬───────┬───────┬───────┬───────┬───────┬───────┬───────┬───────┬───────┐
│ act ┆ act ┆ act ┆ action ┆ actio ┆ actio ┆ clip_ ┆ noun ┆ noun_ ┆ split ┆ verb ┆ verb_ ┆ video │
│ ion ┆ ion ┆ ion ┆ _clip_ ┆ n_cli ┆ n_idx ┆ uid ┆ --- ┆ label ┆ --- ┆ --- ┆ label ┆ _uid │
│ _cl ┆ _cl ┆ _cl ┆ start_ ┆ p_sta ┆ --- ┆ --- ┆ str ┆ --- ┆ str ┆ str ┆ --- ┆ --- │
│ ip_ ┆ ip_ ┆ ip_ ┆ sec ┆ rt_se ┆ i64 ┆ str ┆ ┆ i64 ┆ ┆ ┆ i64 ┆ str │
│ end ┆ end ┆ sta ┆ --- ┆ c_dif ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ │
│ _fr ┆ _se ┆ rt_ ┆ f64 ┆ f ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ │
│ ame ┆ c ┆ fra ┆ ┆ --- ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ │
│ --- ┆ --- ┆ me ┆ ┆ f64 ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ │
│ i64 ┆ f64 ┆ --- ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ │
│ ┆ ┆ i64 ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ │
╞═════╪═════╪═════╪════════╪═══════╪═══════╪═══════╪═══════╪═══════╪═══════╪═══════╪═══════╪═══════╡
│ 47 ┆ 4.7 ┆ 23 ┆ 2.3 ┆ 0.0 ┆ 0 ┆ clip1 ┆ noun2 ┆ 2 ┆ train ┆ verb4 ┆ 4 ┆ video │
│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ 1 │
│ 82 ┆ 8.2 ┆ 39 ┆ 3.9 ┆ 1.6 ┆ 1 ┆ clip1 ┆ noun3 ┆ 3 ┆ train ┆ verb2 ┆ 2 ┆ video │
│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ 1 │
│ 102 ┆ 10. ┆ 74 ┆ 7.4 ┆ 3.5 ┆ 2 ┆ clip1 ┆ noun1 ┆ 1 ┆ train ┆ verb1 ┆ 1 ┆ video │
│ ┆ 2 ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ 1 │
│ 74 ┆ 7.4 ┆ 12 ┆ 1.2 ┆ 0.0 ┆ 0 ┆ clip2 ┆ noun1 ┆ 1 ┆ train ┆ verb1 ┆ 1 ┆ video │
│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ 2 │
│ 142 ┆ 14. ┆ 82 ┆ 8.2 ┆ 7.0 ┆ 1 ┆ clip2 ┆ noun2 ┆ 2 ┆ train ┆ verb2 ┆ 2 ┆ video │
│ ┆ 2 ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ 2 │
└─────┴─────┴─────┴────────┴───────┴───────┴───────┴───────┴───────┴───────┴───────┴───────┴───────┘
>>> metadata
{}
arctix.dataset.ego4d.to_array ¶
to_array(
frame: DataFrame, group_col: str = CLIP_ID
) -> dict[str, ndarray]
Convert a DataFrame to a dictionary of arrays.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
frame
|
DataFrame
|
The input DataFrame. |
required |
group_col
|
str
|
The column used to generate the sequences. |
CLIP_ID
|
Returns:
| Type | Description |
|---|---|
dict[str, ndarray]
|
The dictionary of arrays. |
Example usage:
>>> import polars as pl
>>> from arctix.dataset.ego4d import Column, to_array
>>> frame = pl.DataFrame(
... {
... Column.ACTION_END_FRAME: [47, 82, 102, 74, 142],
... Column.ACTION_END_SEC: [4.7, 8.2, 10.2, 7.4, 14.2],
... Column.ACTION_START_FRAME: [23, 39, 74, 12, 82],
... Column.ACTION_START_SEC: [2.3, 3.9, 7.4, 1.2, 8.2],
... Column.ACTION_START_SEC_DIFF: [0.0, 1.6, 3.5, 0.0, 7.0],
... Column.ACTION_INDEX: [0, 1, 2, 0, 1],
... Column.CLIP_ID: ["clip1", "clip1", "clip1", "clip2", "clip2"],
... Column.NOUN: ["noun2", "noun3", "noun1", "noun1", "noun2"],
... Column.NOUN_ID: [2, 3, 1, 1, 2],
... Column.SPLIT: ["train", "train", "train", "train", "train"],
... Column.VERB: ["verb4", "verb2", "verb1", "verb1", "verb2"],
... Column.VERB_ID: [4, 2, 1, 1, 2],
... Column.VIDEO_ID: ["video1", "video1", "video1", "video2", "video2"],
... }
... )
>>> arrays = to_array(frame)
>>> arrays
{'noun': masked_array(
data=[['noun2', 'noun3', 'noun1'],
['noun1', 'noun2', --]],
mask=[[False, False, False],
[False, False, True]],
fill_value='N/A',
dtype='<U5'),
'noun_label': masked_array(
data=[[2, 3, 1],
[1, 2, --]],
mask=[[False, False, False],
[False, False, True]],
fill_value=999999),
'split': array(['train', 'train'], dtype='<U5'),
'sequence_length': array([3, 2]),
'action_clip_start_frame': masked_array(
data=[[23, 39, 74],
[12, 82, --]],
mask=[[False, False, False],
[False, False, True]],
fill_value=999999),
'action_clip_start_sec': masked_array(
data=[[2.3, 3.9, 7.4],
[1.2, 8.2, --]],
mask=[[False, False, False],
[False, False, True]],
fill_value=1e+20),
'action_clip_start_sec_diff': masked_array(
data=[[0.0, 1.6, 3.5],
[0.0, 7.0, --]],
mask=[[False, False, False],
[False, False, True]],
fill_value=1e+20),
'action_clip_end_frame': masked_array(
data=[[47, 82, 102],
[74, 142, --]],
mask=[[False, False, False],
[False, False, True]],
fill_value=999999),
'action_clip_end_sec': masked_array(
data=[[4.7, 8.2, 10.2],
[7.4, 14.2, --]],
mask=[[False, False, False],
[False, False, True]],
fill_value=1e+20),
'verb': masked_array(
data=[['verb4', 'verb2', 'verb1'],
['verb1', 'verb2', --]],
mask=[[False, False, False],
[False, False, True]],
fill_value='N/A',
dtype='<U5'),
'verb_label': masked_array(
data=[[4, 2, 1],
[1, 2, --]],
mask=[[False, False, False],
[False, False, True]],
fill_value=999999),
'clip_uid': array(['clip1', 'clip2'], dtype='<U5')}
arctix.dataset.ego4d.to_list ¶
to_list(
frame: DataFrame, group_col: str = CLIP_ID
) -> dict[str, list]
Convert a DataFrame to a dictionary of lists.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
frame
|
DataFrame
|
The input DataFrame. |
required |
group_col
|
str
|
The column used to generate the sequences. |
CLIP_ID
|
Returns:
| Type | Description |
|---|---|
dict[str, list]
|
The dictionary of lists. |
Example usage:
>>> import polars as pl
>>> from arctix.dataset.ego4d import Column, to_list
>>> frame = pl.DataFrame(
... {
... Column.ACTION_END_FRAME: [47, 82, 102, 74, 142],
... Column.ACTION_END_SEC: [4.7, 8.2, 10.2, 7.4, 14.2],
... Column.ACTION_START_FRAME: [23, 39, 74, 12, 82],
... Column.ACTION_START_SEC: [2.3, 3.9, 7.4, 1.2, 8.2],
... Column.ACTION_START_SEC_DIFF: [0.0, 1.6, 3.5, 0.0, 7.0],
... Column.ACTION_INDEX: [0, 1, 2, 0, 1],
... Column.CLIP_ID: ["clip1", "clip1", "clip1", "clip2", "clip2"],
... Column.NOUN: ["noun2", "noun3", "noun1", "noun1", "noun2"],
... Column.NOUN_ID: [2, 3, 1, 1, 2],
... Column.SPLIT: ["train", "train", "train", "train", "train"],
... Column.VERB: ["verb4", "verb2", "verb1", "verb1", "verb2"],
... Column.VERB_ID: [4, 2, 1, 1, 2],
... Column.VIDEO_ID: ["video1", "video1", "video1", "video2", "video2"],
... }
... )
>>> data_list = to_list(frame)
>>> data_list
{'action_clip_end_frame': [[47, 82, 102], [74, 142]],
'action_clip_end_sec': [[4.7, 8.2, 10.2], [7.4, 14.2]],
'action_clip_start_frame': [[23, 39, 74], [12, 82]],
'action_clip_start_sec': [[2.3, 3.9, 7.4], [1.2, 8.2]],
'action_clip_start_sec_diff': [[0.0, 1.6, 3.5], [0.0, 7.0]],
'clip_uid': ['clip1', 'clip2'],
'noun': [['noun2', 'noun3', 'noun1'], ['noun1', 'noun2']],
'noun_label': [[2, 3, 1], [1, 2]],
'sequence_length': [3, 2],
'split': ['train', 'train'],
'verb': [['verb4', 'verb2', 'verb1'], ['verb1', 'verb2']],
'verb_label': [[4, 2, 1], [1, 2]]}
arctix.dataset.epic_kitchen_100 ¶
Contain code to download and prepare the EPIC-KITCHENS-100 data.
The following documentation assumes the data are downloaded in the
directory /path/to/data/epic_kitchen_100/.
arctix.dataset.epic_kitchen_100.fetch_data ¶
fetch_data(
path: Path, split: str, force_download: bool = False
) -> tuple[DataFrame, dict]
Download and load the data for EPIC-KITCHENS-100 dataset.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
path
|
Path
|
The path where to store the downloaded data. |
required |
split
|
str
|
The dataset split. |
required |
force_download
|
bool
|
If |
False
|
Returns:
| Type | Description |
|---|---|
tuple[DataFrame, dict]
|
The annotations in a DataFrame and the metadata. |
Example usage:
>>> from pathlib import Path
>>> from arctix.dataset.multithumos import fetch_data
>>> data, metadata = fetch_data(Path("/path/to/data/epic_kitchen_100/")) # doctest: +SKIP
arctix.dataset.epic_kitchen_100.prepare_data ¶
prepare_data(
frame: DataFrame, metadata: dict
) -> tuple[DataFrame, dict]
Prepare the data.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
frame
|
DataFrame
|
The raw DataFrame. |
required |
metadata
|
dict
|
The metadata wich contains the vocabularies to convert verbs and nouns to index. |
required |
Returns:
| Type | Description |
|---|---|
tuple[DataFrame, dict]
|
A tuple containing the prepared data and the metadata. |
Example usage:
>>> import datetime
>>> import polars as pl
>>> from arctix.dataset.epic_kitchen_100 import Column, prepare_data
>>> frame = pl.DataFrame(
... {
... Column.ALL_NOUN_IDS: [[3], [114], [3]],
... Column.ALL_NOUNS: [["door"], ["light"], ["door"]],
... Column.NARRATION: ["open door", "turn on light", "close door"],
... Column.NARRATION_ID: ["P01_01_0", "P01_01_1", "P01_01_2"],
... Column.NARRATION_TIMESTAMP: [
... datetime.time(0, 0, 1, 89000),
... datetime.time(0, 0, 2, 629000),
... datetime.time(0, 0, 5, 349000),
... ],
... Column.NOUN: ["door", "light", "door"],
... Column.NOUN_ID: [3, 114, 3],
... Column.PARTICIPANT_ID: ["P01", "P01", "P01"],
... Column.START_FRAME: [8, 262, 418],
... Column.START_TIMESTAMP: [
... datetime.time(0, 0, 0, 140000),
... datetime.time(0, 0, 4, 370000),
... datetime.time(0, 0, 6, 980000),
... ],
... Column.STOP_FRAME: [202, 370, 569],
... Column.STOP_TIMESTAMP: [
... datetime.time(0, 0, 3, 370000),
... datetime.time(0, 0, 6, 170000),
... datetime.time(0, 0, 9, 490000),
... ],
... Column.VERB: ["open", "turn-on", "close"],
... Column.VERB_ID: [3, 6, 4],
... Column.VIDEO_ID: ["P01_01", "P01_01", "P01_01"],
... },
... schema={
... Column.ALL_NOUN_IDS: pl.List(pl.Int64),
... Column.ALL_NOUNS: pl.List(pl.String),
... Column.NARRATION: pl.String,
... Column.NARRATION_ID: pl.String,
... Column.NARRATION_TIMESTAMP: pl.Time,
... Column.NOUN: pl.String,
... Column.NOUN_ID: pl.Int64,
... Column.PARTICIPANT_ID: pl.String,
... Column.START_FRAME: pl.Int64,
... Column.START_TIMESTAMP: pl.Time,
... Column.STOP_FRAME: pl.Int64,
... Column.STOP_TIMESTAMP: pl.Time,
... Column.VERB: pl.String,
... Column.VERB_ID: pl.Int64,
... Column.VIDEO_ID: pl.String,
... },
... )
>>> data, metadata = prepare_data(frame, metadata={})
>>> with pl.Config(tbl_cols=-1):
... data
...
shape: (3, 18)
┌─────┬─────┬─────┬─────┬─────┬─────┬─────┬─────┬─────┬─────┬─────┬─────┬─────┬─────┬─────┬─────┬─────┬─────┐
│ all ┆ all ┆ nar ┆ nar ┆ nar ┆ nou ┆ nou ┆ par ┆ sta ┆ sta ┆ sta ┆ sta ┆ sto ┆ sto ┆ sto ┆ ver ┆ ver ┆ vid │
│ _no ┆ _no ┆ rat ┆ rat ┆ rat ┆ n ┆ n_c ┆ tic ┆ rt_ ┆ rt_ ┆ rt_ ┆ rt_ ┆ p_f ┆ p_t ┆ p_t ┆ b ┆ b_c ┆ eo_ │
│ un_ ┆ uns ┆ ion ┆ ion ┆ ion ┆ --- ┆ las ┆ ipa ┆ fra ┆ tim ┆ tim ┆ tim ┆ ram ┆ ime ┆ ime ┆ --- ┆ las ┆ id │
│ cla ┆ --- ┆ --- ┆ _id ┆ _ti ┆ str ┆ s ┆ nt_ ┆ me ┆ e_s ┆ e_s ┆ est ┆ e ┆ _se ┆ sta ┆ str ┆ s ┆ --- │
│ sse ┆ lis ┆ str ┆ --- ┆ mes ┆ ┆ --- ┆ id ┆ --- ┆ eco ┆ eco ┆ amp ┆ --- ┆ con ┆ mp ┆ ┆ --- ┆ str │
│ s ┆ t[s ┆ ┆ str ┆ tam ┆ ┆ i64 ┆ --- ┆ i64 ┆ nd ┆ nd_ ┆ --- ┆ i64 ┆ d ┆ --- ┆ ┆ i64 ┆ │
│ --- ┆ tr] ┆ ┆ ┆ p ┆ ┆ ┆ str ┆ ┆ --- ┆ dif ┆ tim ┆ ┆ --- ┆ tim ┆ ┆ ┆ │
│ lis ┆ ┆ ┆ ┆ --- ┆ ┆ ┆ ┆ ┆ f64 ┆ f ┆ e ┆ ┆ f64 ┆ e ┆ ┆ ┆ │
│ t[i ┆ ┆ ┆ ┆ tim ┆ ┆ ┆ ┆ ┆ ┆ --- ┆ ┆ ┆ ┆ ┆ ┆ ┆ │
│ 64] ┆ ┆ ┆ ┆ e ┆ ┆ ┆ ┆ ┆ ┆ f64 ┆ ┆ ┆ ┆ ┆ ┆ ┆ │
╞═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╡
│ [3] ┆ ["d ┆ ope ┆ P01 ┆ 00: ┆ doo ┆ 3 ┆ P01 ┆ 8 ┆ 0.1 ┆ 0.0 ┆ 00: ┆ 202 ┆ 3.3 ┆ 00: ┆ ope ┆ 3 ┆ P01 │
│ ┆ oor ┆ n ┆ _01 ┆ 00: ┆ r ┆ ┆ ┆ ┆ 4 ┆ ┆ 00: ┆ ┆ 7 ┆ 00: ┆ n ┆ ┆ _01 │
│ ┆ "] ┆ doo ┆ _0 ┆ 01. ┆ ┆ ┆ ┆ ┆ ┆ ┆ 00. ┆ ┆ ┆ 03. ┆ ┆ ┆ │
│ ┆ ┆ r ┆ ┆ 089 ┆ ┆ ┆ ┆ ┆ ┆ ┆ 140 ┆ ┆ ┆ 370 ┆ ┆ ┆ │
│ [11 ┆ ["l ┆ tur ┆ P01 ┆ 00: ┆ lig ┆ 114 ┆ P01 ┆ 262 ┆ 4.3 ┆ 4.2 ┆ 00: ┆ 370 ┆ 6.1 ┆ 00: ┆ tur ┆ 6 ┆ P01 │
│ 4] ┆ igh ┆ n ┆ _01 ┆ 00: ┆ ht ┆ ┆ ┆ ┆ 7 ┆ 3 ┆ 00: ┆ ┆ 7 ┆ 00: ┆ n-o ┆ ┆ _01 │
│ ┆ t"] ┆ on ┆ _1 ┆ 02. ┆ ┆ ┆ ┆ ┆ ┆ ┆ 04. ┆ ┆ ┆ 06. ┆ n ┆ ┆ │
│ ┆ ┆ lig ┆ ┆ 629 ┆ ┆ ┆ ┆ ┆ ┆ ┆ 370 ┆ ┆ ┆ 170 ┆ ┆ ┆ │
│ ┆ ┆ ht ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ │
│ [3] ┆ ["d ┆ clo ┆ P01 ┆ 00: ┆ doo ┆ 3 ┆ P01 ┆ 418 ┆ 6.9 ┆ 2.6 ┆ 00: ┆ 569 ┆ 9.4 ┆ 00: ┆ clo ┆ 4 ┆ P01 │
│ ┆ oor ┆ se ┆ _01 ┆ 00: ┆ r ┆ ┆ ┆ ┆ 8 ┆ 1 ┆ 00: ┆ ┆ 9 ┆ 00: ┆ se ┆ ┆ _01 │
│ ┆ "] ┆ doo ┆ _2 ┆ 05. ┆ ┆ ┆ ┆ ┆ ┆ ┆ 06. ┆ ┆ ┆ 09. ┆ ┆ ┆ │
│ ┆ ┆ r ┆ ┆ 349 ┆ ┆ ┆ ┆ ┆ ┆ ┆ 980 ┆ ┆ ┆ 490 ┆ ┆ ┆ │
└─────┴─────┴─────┴─────┴─────┴─────┴─────┴─────┴─────┴─────┴─────┴─────┴─────┴─────┴─────┴─────┴─────┴─────┘
>>> metadata
{}
arctix.dataset.epic_kitchen_100.to_array ¶
to_array(frame: DataFrame) -> dict[str, ndarray]
Convert a DataFrame to a dictionary of arrays.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
frame
|
DataFrame
|
The input DataFrame. |
required |
Returns:
| Type | Description |
|---|---|
dict[str, ndarray]
|
The dictionary of arrays. |
arctix.dataset.epic_kitchen_100.to_list ¶
to_list(frame: DataFrame) -> dict[str, list]
Convert a DataFrame to a dictionary of lists.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
frame
|
DataFrame
|
The input DataFrame. |
required |
Returns:
| Type | Description |
|---|---|
dict[str, list]
|
The dictionary of lists. |
arctix.dataset.multithumos ¶
Contain code to prepare/preprocess the MultiTHUMOS data.
The following documentation assumes the data are downloaded in the
directory /path/to/data/multithumos/.
arctix.dataset.multithumos.fetch_data ¶
fetch_data(
path: Path, force_download: bool = False
) -> DataFrame
Download and load the data for Breakfast dataset.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
path
|
Path
|
The path where to store the downloaded data. |
required |
force_download
|
bool
|
If |
False
|
Returns:
| Type | Description |
|---|---|
DataFrame
|
The data in a DataFrame |
Example usage:
>>> from pathlib import Path
>>> from arctix.dataset.multithumos import fetch_data
>>> data = fetch_data(Path("/path/to/data/multithumos/")) # doctest: +SKIP
arctix.dataset.multithumos.prepare_data ¶
prepare_data(
frame: DataFrame, split: str = "all"
) -> tuple[DataFrame, dict]
Prepare the data.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
frame
|
DataFrame
|
The raw DataFrame. |
required |
split
|
str
|
The dataset split. By default, the union of all the dataset splits is used. |
'all'
|
Returns:
| Type | Description |
|---|---|
tuple[DataFrame, dict]
|
A tuple containing the prepared data and the metadata. |
Example usage:
>>> import polars as pl
>>> from arctix.dataset.multithumos import Column, prepare_data
>>> frame = pl.DataFrame(
... {
... Column.VIDEO: [
... "video_validation_1",
... "video_test_2",
... "video_validation_1",
... "video_test_2",
... ],
... Column.START_TIME: [72.80, 44.00, 1.50, 17.57],
... Column.END_TIME: [76.40, 50.90, 5.40, 18.33],
... Column.ACTION: ["dribble", "dribble", "dribble", "guard"],
... }
... )
>>> data, metadata = prepare_data(frame)
>>> data
shape: (4, 7)
┌─────────┬───────────┬──────────┬────────────┬────────────┬─────────────────┬────────────────────┐
│ action ┆ action_id ┆ end_time ┆ split ┆ start_time ┆ start_time_diff ┆ video │
│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │
│ str ┆ i64 ┆ f64 ┆ str ┆ f64 ┆ f64 ┆ str │
╞═════════╪═══════════╪══════════╪════════════╪════════════╪═════════════════╪════════════════════╡
│ guard ┆ 1 ┆ 18.33 ┆ test ┆ 17.57 ┆ 0.0 ┆ video_test_2 │
│ dribble ┆ 0 ┆ 50.9 ┆ test ┆ 44.0 ┆ 26.43 ┆ video_test_2 │
│ dribble ┆ 0 ┆ 5.4 ┆ validation ┆ 1.5 ┆ 0.0 ┆ video_validation_1 │
│ dribble ┆ 0 ┆ 76.4 ┆ validation ┆ 72.8 ┆ 71.3 ┆ video_validation_1 │
└─────────┴───────────┴──────────┴────────────┴────────────┴─────────────────┴────────────────────┘
>>> metadata
{'vocab_action': Vocabulary(
counter=Counter({'dribble': 3, 'guard': 1}),
index_to_token=('dribble', 'guard'),
token_to_index={'dribble': 0, 'guard': 1},
)}
arctix.dataset.multithumos.to_array ¶
to_array(frame: DataFrame) -> dict[str, ndarray]
Convert a DataFrame to a dictionary of arrays.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
frame
|
DataFrame
|
The input DataFrame. |
required |
Returns:
| Type | Description |
|---|---|
dict[str, ndarray]
|
The dictionary of arrays. |
Example usage:
>>> import polars as pl
>>> from arctix.dataset.multithumos import Column, to_array
>>> frame = pl.DataFrame(
... {
... Column.VIDEO: [
... "video_validation_1",
... "video_validation_1",
... "video_validation_1",
... "video_validation_2",
... "video_validation_2",
... "video_validation_2",
... "video_validation_2",
... ],
... Column.START_TIME: [1.0, 17.0, 79.0, 2.0, 4.0, 20.0, 27.0],
... Column.START_TIME_DIFF: [0.0, 16.07, 61.73, 0.0, 1.57, 15.68, 7.20],
... Column.END_TIME: [5.0, 18.0, 83.0, 3.0, 5.0, 20.0, 30.0],
... Column.ACTION: [
... "dribble",
... "guard",
... "dribble",
... "guard",
... "guard",
... "guard",
... "shoot",
... ],
... Column.ACTION_ID: [1, 0, 1, 0, 0, 0, 2],
... Column.SPLIT: [
... "validation",
... "validation",
... "validation",
... "validation",
... "validation",
... "validation",
... "validation",
... ],
... },
... )
>>> arrays = to_array(frame)
>>> arrays
{'action': masked_array(
data=[['dribble', 'guard', 'dribble', --],
['guard', 'guard', 'guard', 'shoot']],
mask=[[False, False, False, True],
[False, False, False, False]],
fill_value='N/A',
dtype='<U7'),
'action_id': masked_array(
data=[[1, 0, 1, --],
[0, 0, 0, 2]],
mask=[[False, False, False, True],
[False, False, False, False]],
fill_value=999999),
'end_time': masked_array(
data=[[5.0, 18.0, 83.0, --],
[3.0, 5.0, 20.0, 30.0]],
mask=[[False, False, False, True],
[False, False, False, False]],
fill_value=1e+20),
'sequence_length': array([3, 4]),
'split': array(['validation', 'validation'], dtype='<U10'),
'start_time': masked_array(
data=[[1.0, 17.0, 79.0, --],
[2.0, 4.0, 20.0, 27.0]],
mask=[[False, False, False, True],
[False, False, False, False]],
fill_value=1e+20),
'start_time_diff': masked_array(
data=[[0.0, 16.07, 61.73, --],
[0.0, 1.57, 15.68, 7.2]],
mask=[[False, False, False, True],
[False, False, False, False]],
fill_value=1e+20)}
arctix.dataset.multithumos.to_list ¶
to_list(frame: DataFrame) -> dict[str, list]
Convert a DataFrame to a dictionary of lists.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
frame
|
DataFrame
|
The input DataFrame. |
required |
Returns:
| Type | Description |
|---|---|
dict[str, list]
|
The dictionary of lists. |
Example usage:
>>> import polars as pl
>>> from arctix.dataset.multithumos import Column, to_list
>>> frame = pl.DataFrame(
... {
... Column.VIDEO: [
... "video_validation_1",
... "video_validation_1",
... "video_validation_1",
... "video_validation_2",
... "video_validation_2",
... "video_validation_2",
... "video_validation_2",
... ],
... Column.START_TIME: [1.0, 17.0, 79.0, 2.0, 4.0, 20.0, 27.0],
... Column.START_TIME_DIFF: [0.0, 16.07, 61.73, 0.0, 1.57, 15.68, 7.20],
... Column.END_TIME: [5.0, 18.0, 83.0, 3.0, 5.0, 20.0, 30.0],
... Column.ACTION: [
... "dribble",
... "guard",
... "dribble",
... "guard",
... "guard",
... "guard",
... "shoot",
... ],
... Column.ACTION_ID: [1, 0, 1, 0, 0, 0, 2],
... Column.SPLIT: [
... "validation",
... "validation",
... "validation",
... "validation",
... "validation",
... "validation",
... "validation",
... ],
... },
... )
>>> data_list = to_list(frame)
>>> data_list
{'action': [['dribble', 'guard', 'dribble'], ['guard', 'guard', 'guard', 'shoot']],
'action_id': [[1, 0, 1], [0, 0, 0, 2]],
'end_time': [[5.0, 18.0, 83.0], [3.0, 5.0, 20.0, 30.0]],
'sequence_length': [3, 4],
'split': ['validation', 'validation'],
'start_time': [[1.0, 17.0, 79.0], [2.0, 4.0, 20.0, 27.0]],
'start_time_diff': [[0.0, 16.07, 61.73], [0.0, 1.57, 15.68, 7.2]],
'video': ['video_validation_1', 'video_validation_2']}