Import libraries¶

In [ ]:

import os
import sys
from pathlib import Path
import numpy as np
import pandas as pd


# Resolve the repo root by walking up to the pyproject.toml marker, so this
# notebook works no matter which directory it is launched from.
repo_path = str(next(p for p in [Path.cwd(), *Path.cwd().parents] if (p / "pyproject.toml").exists()))
CHECKOUT_PATH = repo_path
DATASET_PATH = os.path.join(repo_path, "datasets")

os.chdir(CHECKOUT_PATH)
sys.path.insert(0, CHECKOUT_PATH)

import os import sys from pathlib import Path import numpy as np import pandas as pd # Resolve the repo root by walking up to the pyproject.toml marker, so this # notebook works no matter which directory it is launched from. repo_path = str(next(p for p in [Path.cwd(), *Path.cwd().parents] if (p / "pyproject.toml").exists())) CHECKOUT_PATH = repo_path DATASET_PATH = os.path.join(repo_path, "datasets") os.chdir(CHECKOUT_PATH) sys.path.insert(0, CHECKOUT_PATH)

In [ ]:

# import visualization packages
import matplotlib.pyplot as plt
import seaborn as sns

from alinemol.utils import eval_roc_auc, eval_pr_auc, eval_acc, rescale, compute_linear_fit
from alinemol.utils import plot_ID_OOD, plot_ID_OOD_sns


light_color = plt.get_cmap("plasma").colors[170]
# dark_color = plt.get_cmap("plasma").colors[5]
dark_color = "black"


# matplotlib.use("pgf")

"""
matplotlib.rcParams.update(
    {
        "pgf.texsystem": "pdflatex",
        "font.family": "serif",
        "font.serif": "Computer Modern Roman",
        "font.size": 20,
        "text.usetex": True,
        "pgf.rcfonts": False,
    }
)
"""

# import visualization packages import matplotlib.pyplot as plt import seaborn as sns from alinemol.utils import eval_roc_auc, eval_pr_auc, eval_acc, rescale, compute_linear_fit from alinemol.utils import plot_ID_OOD, plot_ID_OOD_sns light_color = plt.get_cmap("plasma").colors[170] # dark_color = plt.get_cmap("plasma").colors[5] dark_color = "black" # matplotlib.use("pgf") """ matplotlib.rcParams.update( { "pgf.texsystem": "pdflatex", "font.family": "serif", "font.serif": "Computer Modern Roman", "font.size": 20, "text.usetex": True, "pgf.rcfonts": False, } ) """

Load Data and Split into Train and Test Sets¶

In [ ]:

DF_PATH = os.path.join(DATASET_PATH, "TDC", "HIV", "HIV_simplified.csv")
df = pd.read_csv(DF_PATH)
df.shape

DF_PATH = os.path.join(DATASET_PATH, "TDC", "HIV", "HIV_simplified.csv") df = pd.read_csv(DF_PATH) df.shape

In [ ]:

## Making HIV dataset balanced
# df_balanced = pd.concat([df[df['label'] == 0].sample(2000), df[df['label'] == 1]]).sample(frac=1)
# df_balanced.to_csv(os.path.join(DATASET_PATH, "MoleculeNet", "HIV", "HIV_balanced.csv"), index=False)

## Making HIV dataset balanced # df_balanced = pd.concat([df[df['label'] == 0].sample(2000), df[df['label'] == 1]]).sample(frac=1) # df_balanced.to_csv(os.path.join(DATASET_PATH, "MoleculeNet", "HIV", "HIV_balanced.csv"), index=False)

In [ ]:

# df = pd.read_csv(os.path.join(DATASET_PATH, "MoleculeNet", "HIV", "HIV_balanced.csv"))

# df = pd.read_csv(os.path.join(DATASET_PATH, "MoleculeNet", "HIV", "HIV_balanced.csv"))

In [ ]:

datset_category = "TDC"
dataset_name = "HIV"
labels, counts = np.unique(df["label"], return_counts=True)
plt.figure(figsize=(6, 4))
plt.bar(labels, counts, align="center", width=0.4, color=light_color, edgecolor=dark_color)
plt.xlabel("Label")
plt.ylabel("Count")
plt.title(f"{dataset_name} Dataset ({datset_category})")
plt.xticks([0, 1], ["Inactive", "Active"])
plt.grid(True, axis="y")

# plt.savefig(os.path.join(repo_path, "assets", f"{dataset_name}_dataset.pdf"), bbox_inches='tight', backend='pgf')
# plt.savefig(os.path.join(repo_path, "assets", "hiv_dataset.svg"), bbox_inches='tight')
plt.show()

datset_category = "TDC" dataset_name = "HIV" labels, counts = np.unique(df["label"], return_counts=True) plt.figure(figsize=(6, 4)) plt.bar(labels, counts, align="center", width=0.4, color=light_color, edgecolor=dark_color) plt.xlabel("Label") plt.ylabel("Count") plt.title(f"{dataset_name} Dataset ({datset_category})") plt.xticks([0, 1], ["Inactive", "Active"]) plt.grid(True, axis="y") # plt.savefig(os.path.join(repo_path, "assets", f"{dataset_name}_dataset.pdf"), bbox_inches='tight', backend='pgf') # plt.savefig(os.path.join(repo_path, "assets", "hiv_dataset.svg"), bbox_inches='tight') plt.show()

In [ ]:

df["label"].value_counts(normalize=False)

df["label"].value_counts(normalize=False)

In [ ]:

X = np.array(df["smiles"])
y = np.array(df["label"])

X = np.array(df["smiles"]) y = np.array(df["label"])

In [ ]:

# split_type="sphere_exclusion"
# hopts = {metrics: str = 'euclidean', distance_cutoff: float = 0.5}

# split_type="random"
# hopts={}

# split_type="scaffold"
# hopts={include_chirality: bool = False}

# split_type="kmeans"
# hopts={n_clusters=10, n_init=10}

# split_type="dbscan"
# hopts= {eps=0.5, metric="euclidean}"

# split_type="optisim"
# hopts= {n_clusters=10, max_subsample_size = 1000, distance_cutoff=0.1}"

# split_type="molecular_weight"
# hopts={}

# split_type="sphere_exclusion" # hopts = {metrics: str = 'euclidean', distance_cutoff: float = 0.5} # split_type="random" # hopts={} # split_type="scaffold" # hopts={include_chirality: bool = False} # split_type="kmeans" # hopts={n_clusters=10, n_init=10} # split_type="dbscan" # hopts= {eps=0.5, metric="euclidean}" # split_type="optisim" # hopts= {n_clusters=10, max_subsample_size = 1000, distance_cutoff=0.1}" # split_type="molecular_weight" # hopts={}

In [ ]:

dataset_category = "TDC"
dataset_names = "CYP2C19"
split_type = "scaffold"
# filename="external_test.csv"
SPLIT_PATH = os.path.join(DATASET_PATH, dataset_category, dataset_names, "split", split_type)
RESULTS_PATH = os.path.join(repo_path, "classification_results", dataset_category, dataset_names, split_type)

# train, external_test = split_molecules_train_test(df, sampler=split_type, train_size=0.9, random_state=42)

dataset_category = "TDC" dataset_names = "CYP2C19" split_type = "scaffold" # filename="external_test.csv" SPLIT_PATH = os.path.join(DATASET_PATH, dataset_category, dataset_names, "split", split_type) RESULTS_PATH = os.path.join(repo_path, "classification_results", dataset_category, dataset_names, split_type) # train, external_test = split_molecules_train_test(df, sampler=split_type, train_size=0.9, random_state=42)

In [ ]:

print(external_test.shape)
print(train.shape)

print(external_test.shape) print(train.shape)

In [ ]:

print(train["label"].sum() / train["label"].shape[0])
print(external_test["label"].sum() / external_test["label"].shape[0])

print(train["label"].sum() / train["label"].shape[0]) print(external_test["label"].sum() / external_test["label"].shape[0])

In [ ]:

df["label"].shape

df["label"].shape

In [ ]:

# train, val, test = split_molecules_train_val_test(train, train_size=0.7, val_size=0.15, sampler="random", random_state=42)

# train, val, test = split_molecules_train_val_test(train, train_size=0.7, val_size=0.15, sampler="random", random_state=42)

In [ ]:

# print(train.shape, val.shape, test.shape, external_test.shape)

# print(train.shape, val.shape, test.shape, external_test.shape)

In [ ]:

train.to_csv(os.path.join(SPLIT_PATH, split_type, "train.csv"), index=False)
external_test.to_csv(os.path.join(SPLIT_PATH, split_type, "external_test.csv"), index=False)

train.to_csv(os.path.join(SPLIT_PATH, split_type, "train.csv"), index=False) external_test.to_csv(os.path.join(SPLIT_PATH, split_type, "external_test.csv"), index=False)

In [ ]:

pd.read_csv(os.path.join(SPLIT_PATH, split_type, "train.csv"))

pd.read_csv(os.path.join(SPLIT_PATH, split_type, "train.csv"))

In [ ]:

from sklearn.metrics import roc_auc_score, average_precision_score, roc_curve

from sklearn.metrics import roc_auc_score, average_precision_score, roc_curve

In [ ]:

test_df = pd.read_csv(os.path.join(SPLIT_PATH, split_type, "external_test.csv"))
predction_df = pd.read_csv(os.path.join(repo_path, "classification_inference_results", "prediction.csv"))

test_df = pd.read_csv(os.path.join(SPLIT_PATH, split_type, "external_test.csv")) predction_df = pd.read_csv(os.path.join(repo_path, "classification_inference_results", "prediction.csv"))

In [ ]:

print(roc_auc_score(test_df["label"], predction_df["label"]))
print(average_precision_score(test_df["label"], predction_df["label"]))

print(roc_auc_score(test_df["label"], predction_df["label"])) print(average_precision_score(test_df["label"], predction_df["label"]))

In [ ]:

test_df[(test_df.smiles != predction_df.canonical_smiles)]

test_df[(test_df.smiles != predction_df.canonical_smiles)]

In [ ]:

predction_df[(test_df.smiles != predction_df.canonical_smiles)]

predction_df[(test_df.smiles != predction_df.canonical_smiles)]

In [ ]:

predction_df

predction_df

In [ ]:

roc_curve(test_df["label"], predction_df["label"])

roc_curve(test_df["label"], predction_df["label"])

In [ ]:

df1 = pd.read_csv(os.path.join(SPLIT_PATH, "test_0.csv"))
df2 = pd.read_csv(os.path.join(RESULTS_PATH, "GCN", "1", "prediction.csv"))

print(f"ROC-AUC score: {eval_roc_auc(df1, df2)}")
print(f"PR-AUC score: {eval_pr_auc(df1, df2)}")
print(f"Accuracy score: {eval_acc(df1, df2)}")

df1 = pd.read_csv(os.path.join(SPLIT_PATH, "test_0.csv")) df2 = pd.read_csv(os.path.join(RESULTS_PATH, "GCN", "1", "prediction.csv")) print(f"ROC-AUC score: {eval_roc_auc(df1, df2)}") print(f"PR-AUC score: {eval_pr_auc(df1, df2)}") print(f"Accuracy score: {eval_acc(df1, df2)}")

In [ ]:

exteral_test_path = os.path.join(SPLIT_PATH, "scaffold", "external_test.csv")
trained_model_path = os.path.join(repo_path, "classification_results", "HIV", "gin_supervised_contextpred", "1")
inference_result_path = os.path.join(repo_path, "classification_inference_results", "HIV", "gin_supervised_contextpred")

import subprocess

result = subprocess.run(
    [
        "python",
        "scripts/classification_inference.py",
        "-f",
        "exteral_test_path",
        "-sc",
        "smiles",
        "-tp",
        "trained_model_path",
        "-ip",
        "inference_result_path",
        "-s",
    ],
    shell=True,
    stdout=subprocess.PIPE,
    stderr=subprocess.PIPE,
)
print(result)

exteral_test_path = os.path.join(SPLIT_PATH, "scaffold", "external_test.csv") trained_model_path = os.path.join(repo_path, "classification_results", "HIV", "gin_supervised_contextpred", "1") inference_result_path = os.path.join(repo_path, "classification_inference_results", "HIV", "gin_supervised_contextpred") import subprocess result = subprocess.run( [ "python", "scripts/classification_inference.py", "-f", "exteral_test_path", "-sc", "smiles", "-tp", "trained_model_path", "-ip", "inference_result_path", "-s", ], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) print(result)

In [ ]:

trained_model_path

trained_model_path

Investigate Accuracy-on-the-line¶

In [ ]:

# accuracy, roc_auc or pr_auc
# Accuracy
dataset_category = "TDC"
dataset_names = "CYP2C9"
split_type = "scaffold"
filenames = [f"test_{i}.csv" for i in range(0, 10)]
SPLIT_PATH = os.path.join(DATASET_PATH, dataset_category, dataset_names, "split")
RESULTS_PATH = os.path.join(repo_path, "classification_results", dataset_category, dataset_names, split_type)


model_names = [
    "GCN",
    "GAT",
    "Weave",
    "MPNN",
    "AttentiveFP",
    "NF",
    "gin_supervised_contextpred",
    "gin_supervised_edgepred",
    "gin_supervised_masking",
    "gin_supervised_infomax",
    "randomForest",
    "SVM",
    "XGB",
]
# model_names=["GCN"]
ID_test_accuracy = []
OOD_test_accuracy = []

ID_test_roc_auc = []
OOD_test_roc_auc = []

ID_test_pr_auc = []
OOD_test_pr_auc = []

test_size = []

for i in range(0, 10):
    for model_name in model_names:
        df = pd.read_csv(os.path.join(RESULTS_PATH, model_name, str(i + 1), "eval.txt"), sep=":", header=None)
        ID_test_accuracy.append(df.iloc[1, 1])
        ID_test_roc_auc.append(df.iloc[2, 1])
        ID_test_pr_auc.append(df.iloc[3, 1])


for i, filename in enumerate(filenames):
    df1 = pd.read_csv(os.path.join(SPLIT_PATH, split_type, filename))
    print(df1.shape)
    for model_name in model_names:
        df = pd.read_csv(os.path.join(RESULTS_PATH, model_name, str(i + 1), "prediction.csv"))
        print(df.shape)
        OOD_test_accuracy.append(eval_acc(df1, df))
        OOD_test_roc_auc.append(eval_roc_auc(df1, df))
        OOD_test_pr_auc.append(eval_pr_auc(df1, df))
        test_size.append(df1.shape[0])

results = pd.DataFrame(
    {
        "ID_test_accuracy": ID_test_accuracy,
        "OOD_test_accuracy": OOD_test_accuracy,
        "ID_test_roc_auc": ID_test_roc_auc,
        "OOD_test_roc_auc": OOD_test_roc_auc,
        "ID_test_pr_auc": ID_test_pr_auc,
        "OOD_test_pr_auc": OOD_test_pr_auc,
    }
)

results["model"] = [model_name for i in range(0, 10) for model_name in model_names]
results["test_size"] = test_size
results["split"] = split_type
results["dataset"] = dataset_names

# accuracy, roc_auc or pr_auc # Accuracy dataset_category = "TDC" dataset_names = "CYP2C9" split_type = "scaffold" filenames = [f"test_{i}.csv" for i in range(0, 10)] SPLIT_PATH = os.path.join(DATASET_PATH, dataset_category, dataset_names, "split") RESULTS_PATH = os.path.join(repo_path, "classification_results", dataset_category, dataset_names, split_type) model_names = [ "GCN", "GAT", "Weave", "MPNN", "AttentiveFP", "NF", "gin_supervised_contextpred", "gin_supervised_edgepred", "gin_supervised_masking", "gin_supervised_infomax", "randomForest", "SVM", "XGB", ] # model_names=["GCN"] ID_test_accuracy = [] OOD_test_accuracy = [] ID_test_roc_auc = [] OOD_test_roc_auc = [] ID_test_pr_auc = [] OOD_test_pr_auc = [] test_size = [] for i in range(0, 10): for model_name in model_names: df = pd.read_csv(os.path.join(RESULTS_PATH, model_name, str(i + 1), "eval.txt"), sep=":", header=None) ID_test_accuracy.append(df.iloc[1, 1]) ID_test_roc_auc.append(df.iloc[2, 1]) ID_test_pr_auc.append(df.iloc[3, 1]) for i, filename in enumerate(filenames): df1 = pd.read_csv(os.path.join(SPLIT_PATH, split_type, filename)) print(df1.shape) for model_name in model_names: df = pd.read_csv(os.path.join(RESULTS_PATH, model_name, str(i + 1), "prediction.csv")) print(df.shape) OOD_test_accuracy.append(eval_acc(df1, df)) OOD_test_roc_auc.append(eval_roc_auc(df1, df)) OOD_test_pr_auc.append(eval_pr_auc(df1, df)) test_size.append(df1.shape[0]) results = pd.DataFrame( { "ID_test_accuracy": ID_test_accuracy, "OOD_test_accuracy": OOD_test_accuracy, "ID_test_roc_auc": ID_test_roc_auc, "OOD_test_roc_auc": OOD_test_roc_auc, "ID_test_pr_auc": ID_test_pr_auc, "OOD_test_pr_auc": OOD_test_pr_auc, } ) results["model"] = [model_name for i in range(0, 10) for model_name in model_names] results["test_size"] = test_size results["split"] = split_type results["dataset"] = dataset_names

In [ ]:

results

results

In [ ]:

results.to_csv(os.path.join(RESULTS_PATH, "results.csv"), index=False)

results.to_csv(os.path.join(RESULTS_PATH, "results.csv"), index=False)

In [ ]:

print(len(ID_test_accuracy))
print(len(ID_test_roc_auc))
print(len(ID_test_pr_auc))

print(len(OOD_test_accuracy))
print(len(OOD_test_roc_auc))
print(len(OOD_test_pr_auc))

print(len(ID_test_accuracy)) print(len(ID_test_roc_auc)) print(len(ID_test_pr_auc)) print(len(OOD_test_accuracy)) print(len(OOD_test_roc_auc)) print(len(OOD_test_pr_auc))

In [ ]:

probit_results = results.copy()
probit_results["ID_test_accuracy"] = rescale(results["ID_test_accuracy"], "probit")
probit_results["OOD_test_accuracy"] = rescale(results["OOD_test_accuracy"], "probit")
probit_results["ID_test_roc_auc"] = rescale(results["ID_test_roc_auc"], "probit")
probit_results["OOD_test_roc_auc"] = rescale(results["OOD_test_roc_auc"], "probit")
probit_results["ID_test_pr_auc"] = rescale(results["ID_test_pr_auc"], "probit")
probit_results["OOD_test_pr_auc"] = rescale(results["OOD_test_pr_auc"], "probit")

probit_results = results.copy() probit_results["ID_test_accuracy"] = rescale(results["ID_test_accuracy"], "probit") probit_results["OOD_test_accuracy"] = rescale(results["OOD_test_accuracy"], "probit") probit_results["ID_test_roc_auc"] = rescale(results["ID_test_roc_auc"], "probit") probit_results["OOD_test_roc_auc"] = rescale(results["OOD_test_roc_auc"], "probit") probit_results["ID_test_pr_auc"] = rescale(results["ID_test_pr_auc"], "probit") probit_results["OOD_test_pr_auc"] = rescale(results["OOD_test_pr_auc"], "probit")

In [ ]:

print("--------------------results-------------------")
print(compute_linear_fit(results["ID_test_accuracy"].values, results["OOD_test_accuracy"].values))
print(compute_linear_fit(results["ID_test_roc_auc"].values, results["OOD_test_roc_auc"].values))
print(compute_linear_fit(results["ID_test_pr_auc"].values, results["OOD_test_pr_auc"].values))
print("--------------------probit transform-------------------")
print(compute_linear_fit(probit_results["ID_test_accuracy"].values, probit_results["OOD_test_accuracy"].values))
print(compute_linear_fit(probit_results["ID_test_roc_auc"].values, probit_results["OOD_test_roc_auc"].values))
print(compute_linear_fit(probit_results["ID_test_pr_auc"].values, probit_results["OOD_test_pr_auc"].values))

print("--------------------results-------------------") print(compute_linear_fit(results["ID_test_accuracy"].values, results["OOD_test_accuracy"].values)) print(compute_linear_fit(results["ID_test_roc_auc"].values, results["OOD_test_roc_auc"].values)) print(compute_linear_fit(results["ID_test_pr_auc"].values, results["OOD_test_pr_auc"].values)) print("--------------------probit transform-------------------") print(compute_linear_fit(probit_results["ID_test_accuracy"].values, probit_results["OOD_test_accuracy"].values)) print(compute_linear_fit(probit_results["ID_test_roc_auc"].values, probit_results["OOD_test_roc_auc"].values)) print(compute_linear_fit(probit_results["ID_test_pr_auc"].values, probit_results["OOD_test_pr_auc"].values))

In [ ]:

(bias, slope), r2 = compute_linear_fit(results["ID_test_accuracy"].values, results["OOD_test_accuracy"].values)

(bias, slope), r2 = compute_linear_fit(results["ID_test_accuracy"].values, results["OOD_test_accuracy"].values)

In [ ]:

bias, slope, r2

bias, slope, r2

In [ ]:

fig, ax = plt.subplots(1, 1, figsize=(10, 6))
ax.scatter(ID_test_accuracy, OOD_test_accuracy, color=light_color, s=40, edgecolor=dark_color, linewidth=1)
ax.axline((0.6, 0.6), slope=1, linestyle="--", linewidth=2)
ax.axline((0.5, 0.6), (1, 1.1), color=dark_color, linestyle="--")
ax.axline((0.5, 0.4), (1, 0.9), color=dark_color, linestyle="--")

ax.set_title(f"{dataset_names} Dataset ({dataset_category})")
ax.set_xlabel("ID Test Accuracy")
ax.set_ylabel("OOD Test Accuracy")


ax.set_xlim(0.4, 0.9)
ax.set_ylim(0.4, 0.9)

ax.grid(False)
ax.axline((0, bias), slope=slope, color="red", linewidth=2)

# fig.savefig(os.path.join(repo_path, "assets", "hiv_roc_auc_ID_OOD.pdf"), bbox_inches='tight', backend='pgf')
plt.show()

fig, ax = plt.subplots(1, 1, figsize=(10, 6)) ax.scatter(ID_test_accuracy, OOD_test_accuracy, color=light_color, s=40, edgecolor=dark_color, linewidth=1) ax.axline((0.6, 0.6), slope=1, linestyle="--", linewidth=2) ax.axline((0.5, 0.6), (1, 1.1), color=dark_color, linestyle="--") ax.axline((0.5, 0.4), (1, 0.9), color=dark_color, linestyle="--") ax.set_title(f"{dataset_names} Dataset ({dataset_category})") ax.set_xlabel("ID Test Accuracy") ax.set_ylabel("OOD Test Accuracy") ax.set_xlim(0.4, 0.9) ax.set_ylim(0.4, 0.9) ax.grid(False) ax.axline((0, bias), slope=slope, color="red", linewidth=2) # fig.savefig(os.path.join(repo_path, "assets", "hiv_roc_auc_ID_OOD.pdf"), bbox_inches='tight', backend='pgf') plt.show()

In [ ]:

plot_ID_OOD(
    ID_test_accuracy, OOD_test_accuracy, dataset_category=dataset_category, dataset_name=dataset_names, metric="pr_auc"
)

plot_ID_OOD( ID_test_accuracy, OOD_test_accuracy, dataset_category=dataset_category, dataset_name=dataset_names, metric="pr_auc" )

In [ ]:

dffig, ax = plt.subplots(1, 1, figsize=(10, 6))
ax.scatter(ID_test_accuracy, OOD_test_accuracy, color=light_color, s=40, edgecolor=dark_color, linewidth=1)
ax.axline((0.6, 0.6), slope=1)
ax.grid(which="both", color="lightgray", linestyle="-", linewidth=1)

plt.show()

dffig, ax = plt.subplots(1, 1, figsize=(10, 6)) ax.scatter(ID_test_accuracy, OOD_test_accuracy, color=light_color, s=40, edgecolor=dark_color, linewidth=1) ax.axline((0.6, 0.6), slope=1) ax.grid(which="both", color="lightgray", linestyle="-", linewidth=1) plt.show()

In [ ]:

fig, ax = plt.subplots(1, 1, figsize=(10, 6))
sns.set_theme(
    style="whitegrid",
    rc={"text.usetex": True, "pgf.rcfonts": False, "font.serif": "Computer Modern Roman", "font.family": "serif"},
)

results_filter = results[results["model"] == "GCN"]
sns.scatterplot(
    data=results_filter,
    x="ID_test_accuracy",
    y="OOD_test_accuracy",
    hue="model",
    palette="plasma",
    s=40,
    ax=ax,
    edgecolor=dark_color,
    linewidth=1,
)
ax.axline((0.5, 0.5), slope=1, linestyle="--")

ax.set_title(f"{dataset_names} Dataset ({dataset_category})", fontsize=16)
ax.set_xlabel("ID Test Accuracy", fontsize=16)
ax.set_ylabel("OOD Test Accuracy", fontsize=16)

fig, ax = plt.subplots(1, 1, figsize=(10, 6)) sns.set_theme( style="whitegrid", rc={"text.usetex": True, "pgf.rcfonts": False, "font.serif": "Computer Modern Roman", "font.family": "serif"}, ) results_filter = results[results["model"] == "GCN"] sns.scatterplot( data=results_filter, x="ID_test_accuracy", y="OOD_test_accuracy", hue="model", palette="plasma", s=40, ax=ax, edgecolor=dark_color, linewidth=1, ) ax.axline((0.5, 0.5), slope=1, linestyle="--") ax.set_title(f"{dataset_names} Dataset ({dataset_category})", fontsize=16) ax.set_xlabel("ID Test Accuracy", fontsize=16) ax.set_ylabel("OOD Test Accuracy", fontsize=16)

In [ ]:

import seaborn as sns

fig, ax = plt.subplots(1, 1, figsize=(10, 6))
sns.set_theme(
    style="whitegrid",
    rc={"text.usetex": True, "pgf.rcfonts": False, "font.serif": "Computer Modern Roman", "font.family": "serif"},
)

new_results = results[results["ID_test_accuracy"] > 0.6]
sns.scatterplot(
    data=new_results,
    x="ID_test_accuracy",
    y="OOD_test_accuracy",
    hue="model",
    palette="plasma",
    s=40,
    ax=ax,
    edgecolor=dark_color,
    linewidth=0.5,
)
ax.axline((0.6, 0.6), slope=1, linestyle="--")

ax.set_title(f"{dataset_names} Dataset ({dataset_category})", fontsize=18)
ax.set_xlabel("ID Test Accuracy", fontsize=18)
ax.set_ylabel("OOD Test Accuracy", fontsize=18)

import seaborn as sns fig, ax = plt.subplots(1, 1, figsize=(10, 6)) sns.set_theme( style="whitegrid", rc={"text.usetex": True, "pgf.rcfonts": False, "font.serif": "Computer Modern Roman", "font.family": "serif"}, ) new_results = results[results["ID_test_accuracy"] > 0.6] sns.scatterplot( data=new_results, x="ID_test_accuracy", y="OOD_test_accuracy", hue="model", palette="plasma", s=40, ax=ax, edgecolor=dark_color, linewidth=0.5, ) ax.axline((0.6, 0.6), slope=1, linestyle="--") ax.set_title(f"{dataset_names} Dataset ({dataset_category})", fontsize=18) ax.set_xlabel("ID Test Accuracy", fontsize=18) ax.set_ylabel("OOD Test Accuracy", fontsize=18)

In [ ]:

# grouped bar plot of ID_test_accuracy vs OOD_test_accuracy for each model in the results dataframe alongsilde each other
fig, ax = plt.subplots(1, 1, figsize=(14, 6))
sns.set_theme(
    style="whitegrid",
    rc={"text.usetex": True, "pgf.rcfonts": False, "font.serif": "Computer Modern Roman", "font.family": "serif"},
)
new_results.groupby("model").mean().plot(
    kind="bar",
    y=["ID_test_accuracy", "OOD_test_accuracy"],
    ax=ax,
    color=["b", "r"],
    edgecolor="black",
    linewidth=1,
    fontsize=20,
)
ax.grid(False, axis="x")
ax.set_xlabel("Model", fontsize=24)
ax.set_ylabel("Accuracy", fontsize=24)
ax.set_ylim(0.5, 0.8)

# grouped bar plot of ID_test_accuracy vs OOD_test_accuracy for each model in the results dataframe alongsilde each other fig, ax = plt.subplots(1, 1, figsize=(14, 6)) sns.set_theme( style="whitegrid", rc={"text.usetex": True, "pgf.rcfonts": False, "font.serif": "Computer Modern Roman", "font.family": "serif"}, ) new_results.groupby("model").mean().plot( kind="bar", y=["ID_test_accuracy", "OOD_test_accuracy"], ax=ax, color=["b", "r"], edgecolor="black", linewidth=1, fontsize=20, ) ax.grid(False, axis="x") ax.set_xlabel("Model", fontsize=24) ax.set_ylabel("Accuracy", fontsize=24) ax.set_ylim(0.5, 0.8)

In [ ]:

import seaborn as sns

fig, ax = plt.subplots(1, 1, figsize=(10, 6))

sns.scatterplot(
    data=results,
    x="ID_test_roc_auc",
    y="OOD_test_roc_auc",
    hue="model",
    palette="plasma",
    s=40,
    ax=ax,
    edgecolor=dark_color,
    linewidth=0.5,
)
ax.axline((0.6, 0.6), slope=1, linestyle="--")

ax.set_title(f"{dataset_names} Dataset ({dataset_category})")
ax.set_xlabel("ID Test ROC-AUC")
ax.set_ylabel("OOD Test ROC-AUC")

import seaborn as sns fig, ax = plt.subplots(1, 1, figsize=(10, 6)) sns.scatterplot( data=results, x="ID_test_roc_auc", y="OOD_test_roc_auc", hue="model", palette="plasma", s=40, ax=ax, edgecolor=dark_color, linewidth=0.5, ) ax.axline((0.6, 0.6), slope=1, linestyle="--") ax.set_title(f"{dataset_names} Dataset ({dataset_category})") ax.set_xlabel("ID Test ROC-AUC") ax.set_ylabel("OOD Test ROC-AUC")

In [ ]:

import seaborn as sns

fig, ax = plt.subplots(1, 1, figsize=(10, 6))

sns.scatterplot(
    data=results,
    x="ID_test_pr_auc",
    y="OOD_test_pr_auc",
    hue="model",
    palette="plasma",
    s=40,
    ax=ax,
    edgecolor=dark_color,
    linewidth=0.5,
)
ax.axline((0.6, 0.6), slope=1, linestyle="--")

ax.set_title(f"{dataset_names} Dataset ({dataset_category})")
ax.set_xlabel("ID Test PR-AUC")
ax.set_ylabel("OOD Test PR-AUC")

import seaborn as sns fig, ax = plt.subplots(1, 1, figsize=(10, 6)) sns.scatterplot( data=results, x="ID_test_pr_auc", y="OOD_test_pr_auc", hue="model", palette="plasma", s=40, ax=ax, edgecolor=dark_color, linewidth=0.5, ) ax.axline((0.6, 0.6), slope=1, linestyle="--") ax.set_title(f"{dataset_names} Dataset ({dataset_category})") ax.set_xlabel("ID Test PR-AUC") ax.set_ylabel("OOD Test PR-AUC")

In [ ]:

plot_ID_OOD_sns(results)

plot_ID_OOD_sns(results)

In [ ]:

fig, ax = plt.subplots(3, 1, figsize=(12, 18))
sns.set_theme(
    style="whitegrid",
    rc={"text.usetex": True, "pgf.rcfonts": False, "font.serif": "Computer Modern Roman", "font.family": "serif"},
)

fig.suptitle(f"{dataset_names} Dataset ({dataset_category})", fontsize=26, y=0.95)
sns.scatterplot(
    data=results, x="ID_test_accuracy", y="OOD_test_accuracy", hue="model", palette="plasma", s=40, ax=ax[0]
)
ax[0].axline((0.5, 0.5), slope=1, linestyle="--")

ax[0].set_xlabel("ID Test Accuracy", fontsize=16)
ax[0].set_ylabel("OOD Test Accuracy", fontsize=16)

sns.scatterplot(data=results, x="ID_test_roc_auc", y="OOD_test_roc_auc", hue="model", palette="plasma", s=40, ax=ax[1])
ax[1].axline((0.5, 0.5), slope=1, linestyle="--")

ax[1].set_xlabel("ID ROC-AUC", fontsize=16)
ax[1].set_ylabel("OOD ROC-AUC", fontsize=16)

sns.scatterplot(data=results, x="ID_test_pr_auc", y="OOD_test_pr_auc", hue="model", palette="plasma", s=40, ax=ax[2])
ax[2].axline((0.5, 0.5), slope=1, linestyle="--")

ax[2].set_xlabel("ID PR-AUC", fontsize=16)
ax[2].set_ylabel("OOD PR-AUC", fontsize=16)

plt.show()

fig, ax = plt.subplots(3, 1, figsize=(12, 18)) sns.set_theme( style="whitegrid", rc={"text.usetex": True, "pgf.rcfonts": False, "font.serif": "Computer Modern Roman", "font.family": "serif"}, ) fig.suptitle(f"{dataset_names} Dataset ({dataset_category})", fontsize=26, y=0.95) sns.scatterplot( data=results, x="ID_test_accuracy", y="OOD_test_accuracy", hue="model", palette="plasma", s=40, ax=ax[0] ) ax[0].axline((0.5, 0.5), slope=1, linestyle="--") ax[0].set_xlabel("ID Test Accuracy", fontsize=16) ax[0].set_ylabel("OOD Test Accuracy", fontsize=16) sns.scatterplot(data=results, x="ID_test_roc_auc", y="OOD_test_roc_auc", hue="model", palette="plasma", s=40, ax=ax[1]) ax[1].axline((0.5, 0.5), slope=1, linestyle="--") ax[1].set_xlabel("ID ROC-AUC", fontsize=16) ax[1].set_ylabel("OOD ROC-AUC", fontsize=16) sns.scatterplot(data=results, x="ID_test_pr_auc", y="OOD_test_pr_auc", hue="model", palette="plasma", s=40, ax=ax[2]) ax[2].axline((0.5, 0.5), slope=1, linestyle="--") ax[2].set_xlabel("ID PR-AUC", fontsize=16) ax[2].set_ylabel("OOD PR-AUC", fontsize=16) plt.show()

In [ ]:

results

results

In [ ]:

model_name = "Weave"
!cat classification_results/HIV/$model_name/1/eval.txt | head -2 | tail -1 | cut -d : -f 2

model_name = "Weave" !cat classification_results/HIV/$model_name/1/eval.txt | head -2 | tail -1 | cut -d : -f 2

In [ ]:

df = pd.read_csv(os.path.join("classification_results", "HIV", "GCN", "1", "eval.txt"), sep=":", header=None)

df = pd.read_csv(os.path.join("classification_results", "HIV", "GCN", "1", "eval.txt"), sep=":", header=None)

In [ ]:

In [ ]:

from rdkit import Chem
import torch

mol = Chem.MolFromSmiles(df["smiles"][0])

from rdkit import Chem import torch mol = Chem.MolFromSmiles(df["smiles"][0])

In [ ]:

edge_features = []
num_bonds = mol.GetNumBonds()

edge_features = [] num_bonds = mol.GetNumBonds()

In [ ]:

for j, smile in enumerate(df["smiles"]):
    mol = Chem.MolFromSmiles(smile)
    num_bonds = mol.GetNumBonds()
    for i in range(num_bonds):
        bond = mol.GetBondWithIdx(i)
        bond_feature = bond.GetBondType().__str__()
        if bond_feature == "DATIVE":
            print(j)
        edge_features.append(bond_feature)

for j, smile in enumerate(df["smiles"]): mol = Chem.MolFromSmiles(smile) num_bonds = mol.GetNumBonds() for i in range(num_bonds): bond = mol.GetBondWithIdx(i) bond_feature = bond.GetBondType().__str__() if bond_feature == "DATIVE": print(j) edge_features.append(bond_feature)

In [ ]:

df.to_csv(os.path.join(repo_path, "datasets", "MoleculeNet", "HIV", "split", "scaffold", "train.csv"), index=False)

df.to_csv(os.path.join(repo_path, "datasets", "MoleculeNet", "HIV", "split", "scaffold", "train.csv"), index=False)

In [ ]:

from collections import Counter

Counter(edge_features)

from collections import Counter Counter(edge_features)

In [ ]:

edge_features = []
num_bonds = mol.GetNumBonds()
if num_bonds == 0:
    assert self._self_loop, "The molecule has 0 bonds and we should set self._self_loop to True."

# Compute features for each bond
for i in range(num_bonds):
    bond = mol.GetBondWithIdx(i)
    bond_feats = [self._bond_types.index(bond.GetBondType()), self._bond_direction_types.index(bond.GetBondDir())]
    edge_features.extend([bond_feats, bond_feats.copy()])

if self._self_loop:
    self_loop_features = torch.zeros((mol.GetNumAtoms(), 2), dtype=torch.int64)
    self_loop_features[:, 0] = len(self._bond_types)

if num_bonds == 0:
    edge_features = self_loop_features
else:
    edge_features = np.stack(edge_features)
    edge_features = F.zerocopy_from_numpy(edge_features.astype(np.int64))
    if self._self_loop:
        edge_features = torch.cat([edge_features, self_loop_features], dim=0)

edge_features = [] num_bonds = mol.GetNumBonds() if num_bonds == 0: assert self._self_loop, "The molecule has 0 bonds and we should set self._self_loop to True." # Compute features for each bond for i in range(num_bonds): bond = mol.GetBondWithIdx(i) bond_feats = [self._bond_types.index(bond.GetBondType()), self._bond_direction_types.index(bond.GetBondDir())] edge_features.extend([bond_feats, bond_feats.copy()]) if self._self_loop: self_loop_features = torch.zeros((mol.GetNumAtoms(), 2), dtype=torch.int64) self_loop_features[:, 0] = len(self._bond_types) if num_bonds == 0: edge_features = self_loop_features else: edge_features = np.stack(edge_features) edge_features = F.zerocopy_from_numpy(edge_features.astype(np.int64)) if self._self_loop: edge_features = torch.cat([edge_features, self_loop_features], dim=0)

In [ ]:

df = pd.read_csv(os.path.join(DATASET_PATH, "TDC", "CYP2C19", "cyp2c19_veith.tab"), sep="\t")

df = pd.read_csv(os.path.join(DATASET_PATH, "TDC", "CYP2C19", "cyp2c19_veith.tab"), sep="\t")

In [ ]:

SPLIT_PATH = os.path.join(DATASET_PATH, "TDC", "CYP2C19", "split", "scaffold")

SPLIT_PATH = os.path.join(DATASET_PATH, "TDC", "CYP2C19", "split", "scaffold")

In [ ]:

df = df[["Drug", "Y"]]
df.columns = ["smiles", "label"]
df.to_csv(os.path.join(DATASET_PATH, "TDC", "CYP2C19", "CYP2C19_simplified.csv"), index=False)

df = df[["Drug", "Y"]] df.columns = ["smiles", "label"] df.to_csv(os.path.join(DATASET_PATH, "TDC", "CYP2C19", "CYP2C19_simplified.csv"), index=False)

In [ ]:

labels, counts = np.unique(df["label"], return_counts=True)
plt.figure(figsize=(6, 4))
plt.bar(labels, counts, align="center", width=0.4, color=light_color, edgecolor=dark_color)
plt.xlabel("Label")
plt.ylabel("Count")
plt.title("CYP2C19 Dataset (TDC)")
plt.xticks([0, 1], ["Inactive", "Active"])
plt.grid(True, axis="y")

plt.savefig(os.path.join(repo_path, "assets", "cyp2c19_dataset.pdf"), bbox_inches="tight", backend="pgf")
# plt.savefig(os.path.join(repo_path, "assets", "hiv_dataset.svg"), bbox_inches='tight')
plt.show()

labels, counts = np.unique(df["label"], return_counts=True) plt.figure(figsize=(6, 4)) plt.bar(labels, counts, align="center", width=0.4, color=light_color, edgecolor=dark_color) plt.xlabel("Label") plt.ylabel("Count") plt.title("CYP2C19 Dataset (TDC)") plt.xticks([0, 1], ["Inactive", "Active"]) plt.grid(True, axis="y") plt.savefig(os.path.join(repo_path, "assets", "cyp2c19_dataset.pdf"), bbox_inches="tight", backend="pgf") # plt.savefig(os.path.join(repo_path, "assets", "hiv_dataset.svg"), bbox_inches='tight') plt.show()

In [ ]:

df["label"].value_counts(normalize=False)

df["label"].value_counts(normalize=False)

In [ ]:

np.unique(df["label"], return_counts=True)

np.unique(df["label"], return_counts=True)

In [ ]:

from tdc.single_pred import ADME

data = ADME(name="CYP2C19_Veith")
split = data.get_split(method="scaffold")

from tdc.single_pred import ADME data = ADME(name="CYP2C19_Veith") split = data.get_split(method="scaffold")

In [ ]:

SPLIT_PATH = os.path.join(DATASET_PATH, "TDC", "CYP2C19", "split", "scaffold")

external_test = split["test"]
external_test = external_test[["Drug", "Y"]]
external_test.columns = ["smiles", "label"]
external_test.to_csv(os.path.join(SPLIT_PATH, "external_test_TDC.csv"), index=False)

train = pd.concat([split["train"], split["valid"]])
train = train[["Drug", "Y"]]
train.columns = ["smiles", "label"]
train.to_csv(os.path.join(SPLIT_PATH, "train_TDC.csv"), index=False)

SPLIT_PATH = os.path.join(DATASET_PATH, "TDC", "CYP2C19", "split", "scaffold") external_test = split["test"] external_test = external_test[["Drug", "Y"]] external_test.columns = ["smiles", "label"] external_test.to_csv(os.path.join(SPLIT_PATH, "external_test_TDC.csv"), index=False) train = pd.concat([split["train"], split["valid"]]) train = train[["Drug", "Y"]] train.columns = ["smiles", "label"] train.to_csv(os.path.join(SPLIT_PATH, "train_TDC.csv"), index=False)

In [ ]:

fingerprint = "morgan_fingerprint"
fprints_hopts = {
    "radius": 2,
    "n_bits": 2048,
}
from ALineMol.utils.split_utils import compute_similarities, featurize

# external_test = pd.read_csv(os.path.join(SPLIT_PATH, "sphere_exclusion", "external_test.csv"))
# train = pd.read_csv(os.path.join(SPLIT_PATH, "sphere_exclusion", "train.csv"))

# compute_similarities(external_test['smiles'].values, train['smiles'].values, fingerprint=fingerprint, fprints_hopts=fprints_hopts)

fingerprint = "morgan_fingerprint" fprints_hopts = { "radius": 2, "n_bits": 2048, } from ALineMol.utils.split_utils import compute_similarities, featurize # external_test = pd.read_csv(os.path.join(SPLIT_PATH, "sphere_exclusion", "external_test.csv")) # train = pd.read_csv(os.path.join(SPLIT_PATH, "sphere_exclusion", "train.csv")) # compute_similarities(external_test['smiles'].values, train['smiles'].values, fingerprint=fingerprint, fprints_hopts=fprints_hopts)

In [ ]:

from scipy.spatial import distance
from typing import List, Union, Dict


def compute_similarities(
    source_molecules: Union[List, np.ndarray],
    target_molecules: Union[List, np.ndarray],
    fingerprint: str,
    fprints_hopts: Dict,
) -> np.ndarray:
    """
    Compute similarities between two lists of molecules. It receives two lists of
    smiles or RDKit molecule objects, extracts their fingerprints and computes the similarities
    between them.

    Args:
        source_molecules (np.array or list): SMILES strings or RDKit molecule objects.
        target_molecules (np.array or list): SMILES strings or RDKit molecule objects.
        fingerprint (str): The molecular fingerprint to be used.
        fprints_hopts (dict): Hyperparameters for AIMSim.

    Returns:
        np.ndarray: Matrix of similarities between the two lists of molecules
    """
    fps1 = featurize(source_molecules, fingerprint, fprints_hopts)  # assumed train set
    fps2 = featurize(target_molecules, fingerprint, fprints_hopts)  # assumed test set
    sims = 1 - distance.cdist(fps1, fps2, metric="jaccard")
    return sims.astype(np.float32)

from scipy.spatial import distance from typing import List, Union, Dict def compute_similarities( source_molecules: Union[List, np.ndarray], target_molecules: Union[List, np.ndarray], fingerprint: str, fprints_hopts: Dict, ) -> np.ndarray: """ Compute similarities between two lists of molecules. It receives two lists of smiles or RDKit molecule objects, extracts their fingerprints and computes the similarities between them. Args: source_molecules (np.array or list): SMILES strings or RDKit molecule objects. target_molecules (np.array or list): SMILES strings or RDKit molecule objects. fingerprint (str): The molecular fingerprint to be used. fprints_hopts (dict): Hyperparameters for AIMSim. Returns: np.ndarray: Matrix of similarities between the two lists of molecules """ fps1 = featurize(source_molecules, fingerprint, fprints_hopts) # assumed train set fps2 = featurize(target_molecules, fingerprint, fprints_hopts) # assumed test set sims = 1 - distance.cdist(fps1, fps2, metric="jaccard") return sims.astype(np.float32)

In [ ]:

smis = compute_similarities(train["smiles"].values, external_test["smiles"].values, fingerprint, fprints_hopts)

smis = compute_similarities(train["smiles"].values, external_test["smiles"].values, fingerprint, fprints_hopts)

In [ ]:

fig, ax = plt.subplots(1, 1, figsize=(10, 6))
ax.hist(smis.max(axis=0), bins=20, color=light_color, edgecolor=dark_color)
ax.set_xticks(np.arange(0, 1.1, 0.1))
plt.show()

fig, ax = plt.subplots(1, 1, figsize=(10, 6)) ax.hist(smis.max(axis=0), bins=20, color=light_color, edgecolor=dark_color) ax.set_xticks(np.arange(0, 1.1, 0.1)) plt.show()

In [ ]:

compute_similarities(["CCO"], ["CCO"], fingerprint, fprints_hopts)

compute_similarities(["CCO"], ["CCO"], fingerprint, fprints_hopts)