Minor edits

Author: SasCezar

scripts/R/models_stats_dist.R

@@ -15,11 +15,12 @@ df_cso <- read.csv('~/PycharmProjects/SoftwareTopics/data/interim/taxonomy/cso_p
 
 #target <- c("\\# Nodes", "\\# Edges", "\\# Leafs", "\\# Roots", "\\# Self Loops", "\\# Cycles", "\\#  CC", "Pairs Acc", '\\# New Terms', "\\# Parents", "\\# Children", 'Density', 'Avg Eccentricity', 'Diameter', 'Max Parents', 'Max Children', "Avg Depth")
 
-target <- c("\\# Nodes", '\\# New Terms', '\\# Unlinked', "\\# Edges", 'Density',  "\\# Roots", "\\# Leaves", "Avg Parents", "Avg Children", 'Max Parents', 'Max Children', "Avg Depth", 'Diameter' ,"\\# Components", "\\# Loops", "\\# Cycles", 'Is DAG')
+target <- c("\\# Nodes", '\\# New Terms', '\\# Unlinked', "\\# Edges", 'Density',  "\\# Roots", "\\# Leaves", "Avg Parents", "Avg Children", 'Max Parents', 'Max Children', "Avg Depth", 'Diameter' ,"\\# Components", "\\# Loops", "\\# Cycles")
 
 
 df_cso <- df_cso %>% 
   filter(Metric %in% target) %>%
+  mutate(Metric = replace(Metric, Metric == "\\# New Terms", '\\# New Nodes')) %>%
   mutate_at(c('Sim_Threshold', 'Value'), as.numeric)%>%
   mutate(LLM = replace(LLM, LLM == 'all-mpnet-base-v2', 'MP')) %>%
   mutate(LLM = replace(LLM, LLM == 'all-MiniLM-L6-v2', 'L6')) %>%
@@ -39,6 +40,7 @@ df_wiki$Max_Depth <- as.factor(df_wiki$Max_Depth)
 
 df_wiki <- df_wiki %>% 
   filter(Metric %in% target) %>%
+  mutate(Metric = replace(Metric, Metric == "\\# New Terms", '\\# New Nodes')) %>%
   mutate_at(c('Value'), as.numeric) %>%
   unite("processing", cycle:bridge:abstract:minimization, remove = FALSE, sep=', ') %>%
   mutate(Metric = gsub(r"(\\)", "", Metric)) %>%
@@ -54,6 +56,7 @@ df_llm <- read.csv('~/PycharmProjects/SoftwareTopics/data/interim/taxonomy/LLM_p
 
 df_llm <- df_llm %>% 
   filter(Metric %in% target) %>%
+  mutate(Metric = replace(Metric, Metric == "\\# New Terms", '\\# New Nodes')) %>%
   unite("processing", cycle:bridge:abstract:minimization, remove = FALSE, sep=', ') %>%
   mutate(Metric = gsub(r"(\\)", "", Metric)) %>%
   mutate_at(c('Value'), as.numeric) %>%
@@ -68,6 +71,7 @@ df_llm_iter <- read.csv('~/PycharmProjects/SoftwareTopics/data/interim/taxonomy/
 
 df_llm_iter <- df_llm_iter %>% 
   filter(Metric %in% target) %>%
+  mutate(Metric = replace(Metric, Metric == "\\# New Terms", '\\# New Nodes')) %>%
   unite("processing", cycle:bridge:abstract:minimization, remove = FALSE, sep=', ') %>%
   mutate(Metric = gsub(r"(\\)", "", Metric)) %>%
   mutate_at(c('Value'), as.numeric) %>%
@@ -90,7 +94,7 @@ df <- df %>%
   mutate(src = fct_relevel(src, 'CSO', 'Wiki', 'LLM', 'LLM_Iter')) %>%
   mutate(Metric = replace(Metric, Metric == '#  CC', 'Components')) %>%
   mutate(Metric = factor(Metric, 
-                         c("# Nodes", '# New Terms', '# Unlinked', "# Edges", 'Density',  "# Roots", "# Leaves", "Avg Parents", 'Max Parents', "Avg Children", 'Max Children', "Avg Depth", 'Diameter' ,"# Components", "# Loops", "# Cycles", 'Is DAG')))
+                         c("# Nodes", '# New Nodes', '# Unlinked', "# Edges", 'Density',  "# Roots", "# Leaves", "Avg Parents", 'Max Parents', "Avg Children", 'Max Children', "Avg Depth", 'Diameter' ,"# Components", "# Loops", "# Cycles", 'Is DAG')))
 
 #c("# Nodes", "# Edges", "# Leafs", "# Roots", "# Bridges",  "# Intermediate", "# Self Loops", "# Cycles", "Components", "Pairs Acc", '# New Terms', "# Parents", "# Children",  'Max Parents', 'Max Children', 'Density', 'Avg Eccentricity',  'Avg Depth', 'Diameter'))
 

scripts/python/clean.py

@@ -0,0 +1,18 @@
+from entity.taxonomy import Taxonomy
+from processing import CycleRemovalProcessing
+
+def clean():
+    in_path = '/home/sasce/PycharmProjects/SoftwareTopics/src/pipeline/patched_gitranking_ensemble_patched_cascade_topsis_llm_gpt-4-1106-preview_prompt_type_patch.json'
+    out_path = '/home/sasce/PycharmProjects/SoftwareTopics/src/pipeline/patched_gitranking_ensemble_patched_cascade_topsis_llm_gpt-4-1106-preview_prompt_type_patch_final.json'
+
+    taxonomy = Taxonomy().load(in_path)
+    cycle = CycleRemovalProcessing()
+    taxonomy = cycle.process(taxonomy)
+    taxonomy = taxonomy.update()
+    data = taxonomy.model_dump_json(indent=4)
+    with open(out_path, 'w') as f:
+        f.write(data)
+
+
+if __name__ == '__main__':
+    clean()

scripts/python/correct_stats.py

@@ -0,0 +1,36 @@
+import pandas as pd
+
+
+def stats():
+    in_path = '/home/sasce/PycharmProjects/SoftwareTopics/data/result/software_taxonomy_validation_anonymized_merged_w_LLM.csv'
+
+    df = pd.read_csv(in_path)
+
+    """Get count of pairs for each annotator. And also the percentage"""
+    annotator_counts = df.groupby('annotator').size().reset_index(name='counts')
+    annotator_counts = annotator_counts.assign(percentage=annotator_counts['counts'] / len(df) * 100)
+    annotator_counts.to_csv('../../data/result/annotator_counts.csv', index=False)
+
+    """For each annotator get the percentage of is_correct column"""
+    annotator_stats = df.groupby('annotator').agg({'is_correct': 'mean'}).reset_index()
+    annotator_stats.to_csv('../../data/result/annotator_stats.csv', index=False)
+
+    """Drop the rows that have 99 as annotator"""
+    df = df[df['annotator'] != 99]
+
+    """Group by the columns term,hypernym and then sum over the is_correct column and percentage"""
+    df = df.groupby(['term', 'hypernym']).agg({'is_correct': 'sum'}).reset_index()
+    df = df.assign(percentage=df['is_correct'] / len(df) * 100)
+    """If the is_correct column is greater than 3 replace it with ≥3"""
+    df = df.assign(is_correct=df['is_correct'].apply(lambda x: '>3' if x > 3 else x))
+
+    out_path = '../../data/result/annotation_stats.csv'
+    df.to_csv(out_path, index=False)
+
+    """Count each group of is_correct column and percentage"""
+    stats = df.groupby('is_correct').size().reset_index(name='counts')
+    stats = stats.assign(percentage=stats['counts'] / len(df) * 100)
+    stats.to_csv('../../data/result/annotation_stats_counts.csv', index=False)
+
+if __name__ == '__main__':
+    stats()

src/pipeline/evaluation_metrics.py

@@ -40,6 +40,8 @@ def compute_metrics(cfg: DictConfig):
         metrics_rename.update(function.columns_rename)
 
     for folder in folders:
+        if 'temp' not in str(folder):
+            continue
         files = [x for x in folder.iterdir() if x.suffix == '.json']
         metrics = []
         for file in tqdm(files, desc=folder.stem):
@@ -50,7 +52,10 @@ def compute_metrics(cfg: DictConfig):
             pairs = [(x[0], x[1]) for x in taxonomy.pairs]
             g = nx.DiGraph()
             g.add_edges_from(pairs)
-            row = format_params(taxonomy.other['params'])
+            row = format_params(taxonomy.other['params']) if 'params' in taxonomy.other or not taxonomy.other else {'name': file.stem.split('_')[-1]}
+
+            row = {'name': file.stem.split('_')[-1]}
+            print(row)
             logger.info(f"Processing {file.stem}")
             if 'processed' in folder.stem:
                 pp.update(taxonomy.other['post_processing'])
@@ -66,8 +71,9 @@ def compute_metrics(cfg: DictConfig):
             metrics.append(row)
 
         df = pd.DataFrame(metrics)
-        sort_columns = list(format_params(taxonomy.other['params']).keys())
-        df.sort_values(by=sort_columns, ascending=True, inplace=True)
+        if 'params' in taxonomy.other:
+            sort_columns = list(format_params(taxonomy.other['params']).keys())
+            df.sort_values(by=sort_columns, ascending=True, inplace=True)
 
         if 'Sim Threshold' in df.columns:
             df['Sim Threshold'] = df['Sim Threshold'].astype(float)
@@ -89,20 +95,22 @@ def compute_metrics(cfg: DictConfig):
         df.to_csv(folder / 'metrics.csv', index=False)
 
         metrics_name = [x for x in metrics_rename.values() if x in df.columns]
-        header_order = list(format_params(taxonomy.other['params']).keys()) + pp_names + metrics_name
-        df.sort_values(by=list(format_params(taxonomy.other['params']).keys()) + pp_names, ascending=True,
+        if 'params' in taxonomy.other:
+            header_order = list(format_params(taxonomy.other['params']).keys()) + pp_names + metrics_name
+            df.sort_values(by=list(format_params(taxonomy.other['params']).keys()) + pp_names, ascending=True,
                        inplace=True)
 
-        df = df[header_order]
+            df = df[header_order]
         df.to_latex(folder / 'metrics.tex', index=False,
                     escape=False, float_format="{:.2f}".format,
                     header=['\\rot{{' + x + '}}' for x in df.columns])
 
-        id_vars = list(format_params(taxonomy.other['params']).keys()) + pp_names
-        header_rename = {x: str(x).replace(" ", "_") for x in id_vars}
-        df.rename(columns=header_rename, inplace=True)
+        if 'params' in taxonomy.other:
+            id_vars = list(format_params(taxonomy.other['params']).keys()) + pp_names
+            header_rename = {x: str(x).replace(" ", "_") for x in id_vars}
+            df.rename(columns=header_rename, inplace=True)
 
-        melted = df.melt(id_vars=header_rename.values(),
+        melted = df.melt(id_vars='name',
                          value_vars=metrics_name,
                          var_name='Metric', value_name='Value')
         melted.to_csv(folder / 'melted_metrics.csv', index=False)

src/processing/cycle.py

@@ -37,7 +37,7 @@ def process(self, taxonomy: Taxonomy) -> Taxonomy:
         logger.info(f"Number of removed edges: {len(removed)}")
         for loop_edge in removed:
             self.graph.remove_edge(*loop_edge)
-
+        assert nx.is_directed_acyclic_graph(self.graph)
         inverse_exists = [int(self.graph.has_edge(b, a)) for a, b in removed]
         percentage = sum(inverse_exists) / (len(inverse_exists)+1) * 100
         removed = [[a, b, z] for (a, b), z in zip(removed, inverse_exists)]