Upload prediction_multilabel.py

prediction_multilabel.py

@@ -0,0 +1,66 @@
+import time
+import pickle
+import numpy as np
+import pandas as pd
+import torch
+import torch.nn as nn
+from sentence_transformers import util
+
+# Set random seed for reproducibility
+torch.manual_seed(1)
+
+# Load datasets
+df_inmemory = pd.read_csv('df_360k_41lables_05012023.csv')
+df_paragraph = pd.read_csv('german_plc_all_paragraphs_unnested_only.csv')
+
+# Load stored embeddings
+with open('embeddings_paragraphs_07012023.pkl', "rb") as f:
+    stored_data = pickle.load(f)
+    pred_embeddings = stored_data['parg_embeddings']
+
+with open('embeddings_sentences_360k_09012023.pkl', "rb") as f:
+    stored_data = pickle.load(f)
+    embeddings = stored_data['sent_embeddings']
+
+# Define function for cosine similarity search
+def get_top_n_similar_patents(new_claim, claim_embeddings, top_n=20):
+    search_hits = util.semantic_search(new_claim, claim_embeddings, top_k=top_n)
+    top_claim_ids = [hit['corpus_id'] for hit in search_hits[0]]
+    similarity_scores = [hit['score'] for hit in search_hits[0]]
+    
+    return pd.DataFrame({'top_claim_ids': top_claim_ids, 'cosine_similarity': similarity_scores})
+
+# 1. Perform cosine similarity search
+test_embeddings = pred_embeddings[:50000]
+all_predictions = []
+
+start = time.time()
+for i, test_embedding in enumerate(test_embeddings):
+    result_df = get_top_n_similar_patents(test_embedding.reshape(1, -1), embeddings)
+    result = pd.merge(result_df, df_inmemory, left_on='top_claim_ids', right_on='index', how='left')
+    all_predictions.append(result)
+
+df_all_predictions = pd.concat(all_predictions, keys=range(len(all_predictions)), axis=0)
+
+# 2. Apply K-Nearest Neighbor (KNN) algorithm
+top_n = 12
+predict = pd.DataFrame(columns=df_inmemory.columns[6:])
+for item in range(len(all_predictions)):
+    k_similar_patents = df_all_predictions.xs(item).nlargest(top_n, ['cosine_similarity'])
+    result_knn = pd.DataFrame(0, index=[0], columns=k_similar_patents.columns[8:])
+    for i in range(top_n):
+        result_knn += k_similar_patents.iloc[i, 8:].values
+    predict = pd.concat([predict, result_knn], ignore_index=True)
+
+# 3. Apply Sigmoid activation function
+sigmoid = nn.Sigmoid()
+data_tensor = torch.tensor(predict.to_numpy().astype(float), dtype=torch.float32)
+output = sigmoid(data_tensor)
+output = (output > 0.90).float()
+
+# Save results
+output_df = pd.DataFrame(output.numpy(), columns=predict.columns)
+df_results = pd.concat([df_paragraph.iloc[:50000, :].reset_index(), output_df], axis=1)
+df_results.to_csv('df_results_0_50k.csv', index=False)
+
+print(f"Processing completed in {time.time() - start:.2f} seconds.")