In [12]:
e.plot_hierarchical_tree_graph('liquid')
In this notebook, we will examine 4 different applications of SALT using label aggregation methods.
from py_salt import event_mapping
from pprint import pprint as pp
# Init EventExplorer class
e = event_mapping.EventExplorer()
In this example we demonstrate how easy it is to gather labels of specific interest from all the mapped datasets. Let's examine a very fine sound event first: A car horn.
# example for a fine label
dataset_label = 'car_horn'
print(f'Dataset labels for standard label: "{dataset_label}"\n')
pp(e.get_mapping_for_std_label(dataset_label))
Dataset labels for standard label: "car_horn"
{'AudioSet': ['Toot', 'Vehicle horn, car horn, honking'],
'AudioSet_strong': ['Toot', 'Vehicle horn, car horn, honking, toot'],
'ESC50': ['car_horn'],
'Fsd50k': ['Vehicle_horn_and_car_horn_and_honking'],
'MAESTROsynthetic': ['car_horn'],
'SONYC': ['car-horn'],
'Singapura': ['Car horn'],
'UrbanSound8K': ['car horn']}
This utility works for coarse labels as well, allowing the user to gather all labels for a domain of interest from different datasets. Let's say we want to gather data (access to labels) for domestic sound events.
dataset_label = 'domestic_sounds'
print(f'Dataset labels for standard label: "{dataset_label}"\n')
# example for a coarse label
pp(e.get_mapping_for_std_label(std_label=dataset_label))
Dataset labels for standard label: "domestic_sounds"
{'AudioSet': ['Alarm clock',
'Bathtub (filling or washing)',
'Blender',
'Chopping (food)',
'Coin (dropping)',
'Computer keyboard',
'Cupboard open or close',
'Cutlery, silverware',
'Ding-dong',
'Dishes, pots, and pans',
'Domestic sounds, home sounds',
'Door',
'Doorbell',
'Drawer open or close',
'Electric shaver, electric razor',
'Electric toothbrush',
'Frying (food)',
'Hair dryer',
'Kettle whistle',
'Keys jangling',
'Microwave oven',
'Packing tape, duct tape',
'Radio',
'Scissors',
'Shuffling cards',
'Sink (filling or washing)',
'Slam',
'Sliding door',
'Television',
'Toilet flush',
'Toothbrush',
'Typewriter',
'Typing',
'Vacuum cleaner',
'Water tap, faucet',
'Writing',
'Zipper (clothing)'],
'AudioSet_strong': ['Alarm clock',
'Bathroom sounds',
'Bathtub (filling or washing)',
'Blender, food processor',
'Chopping (food)',
'Coin (dropping)',
'Computer keyboard',
'Crockery breaking and smashing',
'Cupboard open or close',
'Cutlery, silverware',
'Ding-dong',
'Dishes, pots, and pans',
'Domestic sounds, home sounds',
'Dong, bong',
'Door',
'Doorbell',
'Drawer open or close',
'Electric shaver, electric razor',
'Electric toothbrush',
'Frying (food)',
'Hair dryer',
'Kettle whistle',
'Keys jangling',
'Kitchen and dining room sounds',
'Knife',
'Microwave oven',
'Packing tape, duct tape',
'Paper rustling',
'Radio',
'Refrigerator',
'Scissors',
'Shower',
'Shuffling cards',
'Sink (filling or washing)',
'Slam',
'Sliding door',
'Stairs',
'Television',
'Toilet flush',
'Toothbrush',
'Typewriter',
'Typing',
'Vacuum cleaner',
'Velcro, hook and loop fastener',
'Video game sound',
'Washing machine',
'Water tap, faucet',
'Writing',
'Zipper (clothing)'],
'CHiMEHome': ['Broadband noise, e.g. household appliances', 'Video game/TV'],
'DESEDReal': ['dishes',
'Electric_shaver_toothbrush',
'Frying',
'Running_water',
'Vacuum_cleaner'],
'ESC50': ['brushing_teeth',
'can_opening',
'door_wood_creaks',
'door_wood_knock',
'keyboard_typing',
'mouse_click',
'toilet_flush',
'vacuum_cleaner',
'washing_machine'],
'Fsd50k': ['Bathtub_(filling_or_washing)',
'Coin_(dropping)',
'Computer_keyboard',
'Cupboard_open_or_close',
'Cutlery_and_silverware',
'Dishes_and_pots_and_pans',
'Domestic_sounds_and_home_sounds',
'Door',
'Doorbell',
'Drawer_open_or_close',
'Frying_(food)',
'Keys_jangling',
'Microwave_oven',
'Packing_tape_and_duct_tape',
'Scissors',
'Sink_(filling_or_washing)',
'Slam',
'Sliding_door',
'Toilet_flush',
'Typewriter',
'Typing',
'Water_tap_and_faucet',
'Writing',
'Zipper_(clothing)'],
'MAESTROreal': ['cutlery and dishes',
'door opens/closes',
'furniture dragging'],
'ReaLISED': ['beater',
'cooking',
'cupboard/wardrobe',
'dishwasher',
'drawer',
'furniture movement',
'microwave',
'object falling',
'smoke extractor',
'switch',
'television',
'vacuum cleaner',
'walking',
'washing machine',
'water tap',
'window'],
'Starss22': ['Domestic sounds',
'Door, open or close',
'Knock',
'Water tap, faucet'],
'TAUNIGENSSpatialSoundEvents2020': ['knocking on door'],
'TUTSoundEvents2016': ['cupboard',
'cutlery',
'dishes',
'drawer',
'washing dishes']}
Let's suppose we are working with data from AudioSet, and we are particularly interested in gathering more data that are associated with the label "Howl (wind)". Below, we demonstrate how we can achieve this, using a simple function:
e.reset_map_df()
# Start with a dataset label, e.g. AudioSet's "Howl (wind)"
dataset_label = 'Howl (wind)'
# Get the mapping dictionary
mapping_dict, std_label = e.get_mapping_for_dataset_label(
dataset_label=dataset_label, return_std_label=True)
print(f'Standard label for "{dataset_label}": "{std_label}"\n')
print('Associated labels from external datasets:\n')
pp(mapping_dict)
Standard label for "Howl (wind)": "wind_blowing"
Associated labels from external datasets:
{'Archeo': ['wind'],
'AudioSet': ['Wind',
'Wind noise (microphone)',
'Howl (wind)',
'Rustling leaves'],
'AudioSet_strong': ['Wind',
'Wind noise (microphone)',
'Howl (wind)',
'Rustling leaves'],
'ESC50': ['wind'],
'Fsd50k': ['Wind'],
'MAESTROreal': ['wind_blowing'],
'Singapura': ['Wind'],
'TUTSoundEvents2016': ['wind blowing']}
In this example, we demonstrate how we can gather data for a whole label "domain" starting from a very fine dataset label. Let's suppose we are working with the label "Hose pump" from SINGA:PURA...
First, let's find the standard label that "Hose pump" is mapped to:
dataset_label = 'Hose pump'
e.get_std_label_from_dataset_label(dataset_label)
'hose_pump'
Expectedly, it's simply the same label using the "Audible" naming convention. Next, let's find the label paths from coarse to fine labels for "hose_pump":
e.get_paths_to_label('hose_pump')
[['sound_of_things', 'liquid', 'hose_pump']]
Interesting! Let's find out what else lies under "liquid"
e.plot_hierarchical_tree_graph('liquid')
Finally, let's get all the dataset labels associated with "liquid":
domain_labels = e.get_mapping_for_std_label('liquid')
pp(domain_labels)
{'AudioSet': ['Boiling',
'Drip',
'Fill (with liquid)',
'Gush',
'Liquid',
'Pour',
'Pump (liquid)',
'Slosh',
'Splash, splatter',
'Spray',
'Squish',
'Stir',
'Trickle, dribble'],
'AudioSet_strong': ['Boiling',
'Drip',
'Fill (with liquid)',
'Gush',
'Liquid',
'Pour',
'Pump (liquid)',
'Slosh',
'Splash, splatter',
'Spray',
'Squish',
'Stir',
'Trickle, dribble'],
'Fsd50k': ['Boiling',
'Drip',
'Fill_(with_liquid)',
'Liquid',
'Pour',
'Splash_and_splatter',
'Trickle_and_dribble'],
'Singapura': ['Hose pump']}
This is how we moved from a simple label (Hose pump), to a whole domain of labels of the same category (liquid)!
In this example, we demonstrate how we can gather the common set of standard labels across different datasets
e.get_mapped_datasets()
['Fsd50k', 'AudioSet_strong', 'AudioSet', 'ESC50', 'Singapura', 'MAESTROreal', 'TUTSoundEvents2017', 'Urbansas', 'MAVDtraffic', 'IDMTtraffic', 'UrbanSound8K', 'TUTSoundEvents2016', 'MAESTROsynthetic', 'SONYC', 'Archeo', 'TAUNIGENSSpatialSoundEvents2020', 'Nigens', 'TUTRareSoundEvents', 'CHiMEHome', 'ReaLISED', 'DESEDReal', 'Starss22', 'Starss23', 'MATS', 'Nonspeech7k', 'AnimalSound']
Let's choose ReaLISED, MATS & CHiMEHome
datasets = ['ReaLISED', 'MATS', 'CHiMEHome']
e.find_datasets_intersection(datasets)
['human_sounds', 'human_voice', 'sound_of_things', 'channel_environment_and_background', 'speech', 'talking']
Another example using Starss22, ESC50, AudioSet & UrbanSound8K. This time, we will also gather the dataset labels for each one of the selected datasets.
datasets = ['Starss22', 'ESC50', 'AudioSet', 'UrbanSound8K']
# Find the common set of standard labels
std_labels = e.find_datasets_intersection(datasets)
print('Standard labels:')
print(std_labels)
print()
# Filter the mapping dataframe through the selected datasets
e.filter_by_datasets(datasets)
# Get dataset labels for each one of the standard labels
dataset_labels = {}
for std_label in std_labels:
dataset_labels.update(e.get_mapping_for_std_label(std_label))
print('Number of dataset labels per dataset:')
for dataset in dataset_labels:
print(f'{dataset}: {len(dataset_labels[dataset])}')
print()
pp(dataset_labels)
Standard labels:
['human_sounds', 'human_voice', 'sound_of_things', 'human_non-speech_sounds', 'alarm_signal']
Number of dataset labels per dataset:
AudioSet: 28
ESC50: 3
UrbanSound8K: 2
Starss22: 1
{'AudioSet': ['Alarm',
'Fire alarm',
'Smoke detector, smoke alarm',
'Alarm clock',
'Car alarm',
'Buzzer',
'Foghorn',
'Whistle',
'Kettle whistle',
'Steam whistle',
'Bicycle bell',
'Air horn, truck horn',
'Toot',
'Vehicle horn, car horn, honking',
'Telephone',
'Telephone dialing, DTMF',
'Telephone bell ringing',
'Busy signal',
'Dial tone',
'Siren',
'Civil defense siren',
'Police car (siren)',
'Ambulance (siren)',
'Fire engine, fire truck (siren)',
'Reversing beeps',
'Doorbell',
'Ding-dong',
'Ringtone'],
'ESC50': ['clock_alarm', 'car_horn', 'siren'],
'Starss22': ['Telephone'],
'UrbanSound8K': ['car horn', 'siren']}