File size: 967 Bytes
5434c4b 6503e4e a8a595d aab3281 5434c4b 6503e4e a8a595d 6503e4e 5434c4b aab3281 0c136d8 a8a595d 5434c4b 5f3a4af a8a595d 5f3a4af |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 |
import diff_match_patch as dmp_module
from api_wrappers import hf_data_loader
def get_annotated_diff(start_text, end_text):
dmp = dmp_module.diff_match_patch()
dmp_mapping = {
-1: '-',
0: None,
1: '+'
}
diff = dmp.diff_main(start_text, end_text)
dmp.diff_cleanupSemantic(diff)
result = [[w, dmp_mapping[t]] for t, w in diff]
return result
def annotated_diff_for_row(row):
start = row['commit_msg_start']
end = row['commit_msg_end']
return get_annotated_diff(start, end)
def manual_data_with_annotated_diffs():
df = hf_data_loader.load_raw_rewriting_dataset_as_pandas()
annotated = df.apply(annotated_diff_for_row, axis=1)
df['annotated_diff'] = annotated
return df
def synthetic_data_with_annotated_diffs():
df = hf_data_loader.load_synthetic_dataset_as_pandas()
annotated = df.apply(annotated_diff_for_row, axis=1)
df['annotated_diff'] = annotated
return df
|