From d86ae81e6a7dc138e607bafc44f6154fc2735a1c Mon Sep 17 00:00:00 2001 From: saeedkhosravi94 Date: Sat, 8 Nov 2025 08:53:32 +0100 Subject: [PATCH] init commit --- DLSTSVM.py | 75 +++++++++++++++++++++++++ DTSVM.py | 79 ++++++++++++++++++++++++++ LSTSVM.py | 100 +++++++++++++++++++++++++++++++++ RULSTSVM.py | 92 +++++++++++++++++++++++++++++++ RUTSVM.py | 156 ++++++++++++++++++++++++++++++++++++++++++++++++++++ TSVM.py | 82 +++++++++++++++++++++++++++ UTSVM.py | 146 ++++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 730 insertions(+) create mode 100755 DLSTSVM.py create mode 100755 DTSVM.py create mode 100755 LSTSVM.py create mode 100755 RULSTSVM.py create mode 100755 RUTSVM.py create mode 100755 TSVM.py create mode 100755 UTSVM.py diff --git a/DLSTSVM.py b/DLSTSVM.py new file mode 100755 index 0000000..a0715e4 --- /dev/null +++ b/DLSTSVM.py @@ -0,0 +1,75 @@ +""" +Article : Deep Least Squares Support Vector Machine +Link : New +Author : Saeed Khosravi +""" +import numpy as np +import LSTSVM +class DLSTSVM: + def __init__(self, X, y, C, eps = 1e-4): + self.X = X + self.y = y + self.C = C + self.eps = eps + + def fit(self): + #LSTSVM 1 + C1 = self.C[0] + C2 = self.C[1] + y = self.y + lstsvm1 = LSTSVM.LSTSVM(self.X, y, C1, C2) + lstsvm1.fit() + self.w11, self.b11, self.w12, self.b12 = lstsvm1.get_params() + self.f1 = self.f_(self.X, self.w11, self.b11, self.w12, self.b12) + + #LSTSVM 2 + C1 = self.C[2] + C2 = self.C[3] + y = self.y + lstsvm2 = LSTSVM.LSTSVM(self.X, y, C1, C2) + lstsvm2.fit() + self.w21, self.b21, self.w22, self.b22 = lstsvm2.get_params() + self.f2 = self.f_(self.X, self.w21, self.b21, self.w22, self.b22) + + #LSTSVM Main + C1 = self.C[4] + C2 = self.C[5] + X = self.f(self.f1, self.f2) + y = self.y + lstsvm_M = LSTSVM.LSTSVM(X, y, C1, C2) + lstsvm_M.fit() + self.w1, self.b1, self.w2, self.b2 = lstsvm_M.get_params() + + def predict(self, x_test, y_test): + f1 = self.f_(x_test, self.w11, self.b11, self.w12, self.b12) + f2 = self.f_(x_test, self.w21, self.b21, self.w22, self.b22) + f = self.f(f1, f2) + distance_1 = np.abs(np.dot(f, self.w1) + self.b1) + distance_2 = np.abs(np.dot(f, self.w2) + self.b2) + y_pred = np.zeros_like(distance_1) + for i in range(y_pred.shape[0]): + if (distance_1[i] < distance_2[i]): + y_pred[i][0] = 1; + else: + y_pred[i][0] = -1; + self.preds = y_pred + + def f_(self, x, w1, b1, w2, b2): + f = np.concatenate((np.dot(x, w1)+b1, np.dot(x, w2)+b2), axis = 1) + return f + + def f(self, f1, f2): + f = np.concatenate((f1,f2), axis = 1) + return f + + + def get_hidden_params(self): + return self.w11, self.b11, self.w12, self.b12, self.w21, self.b21, self.w22, self.b22 + + def get_output_params(self): + return self.w1, self.b1, self.w2, self.b2 + + def get_preds(self): + return self.preds + + \ No newline at end of file diff --git a/DTSVM.py b/DTSVM.py new file mode 100755 index 0000000..17e95b7 --- /dev/null +++ b/DTSVM.py @@ -0,0 +1,79 @@ +""" +Article : Deep Twin Support Vector Machine +Link : https://sci-hub.tw/https://ieeexplore.ieee.org/abstract/document/7022580 +Author : Saeed Khosravi + +""" + +import numpy as np +import TSVM + +class DTSVM: + + def __init__(self, X, y, C, eps = 1e-4): + + self.X = X + self.y = y + self.C = C + self.eps = eps + + def fit(self): + #LSTSVM 1 + C1 = self.C[0] + C2 = self.C[1] + y = self.y + tsvm1 = TSVM.TSVM(self.X, y, C1, C2) + tsvm1.fit() + self.w11, self.b11, self.w12, self.b12 = tsvm1.get_params() + self.f1 = self.f_(self.X, self.w11, self.b11, self.w12, self.b12) + + #LSTSVM 2 + C1 = self.C[2] + C2 = self.C[3] + y = self.y + tsvm2 = TSVM.TSVM(self.X, y, C1, C2) + tsvm2.fit() + self.w21, self.b21, self.w22, self.b22 = tsvm2.get_params() + self.f2 = self.f_(self.X, self.w21, self.b21, self.w22, self.b22) + + #LSTSVM Main + C1 = self.C[4] + C2 = self.C[5] + X = self.f(self.f1, self.f2) + y = self.y + tsvm_M = TSVM.TSVM(X, y, C1, C2) + tsvm_M.fit() + self.w1, self.b1, self.w2, self.b2 = tsvm_M.get_params() + + + def predict(self, x_test, y_test): + f1 = self.f_(x_test, self.w11, self.b11, self.w12, self.b12) + f2 = self.f_(x_test, self.w21, self.b21, self.w22, self.b22) + f = self.f(f1, f2) + distance_1 = np.abs(np.dot(f, self.w1) + self.b1) + distance_2 = np.abs(np.dot(f, self.w2) + self.b2) + y_pred = np.zeros_like(distance_1) + for i in range(y_pred.shape[0]): + if (distance_1[i] < distance_2[i]): + y_pred[i][0] = 1; + else: + y_pred[i][0] = -1; + self.preds = y_pred + + def f_(self, x, w1, b1, w2, b2): + f = np.concatenate((np.dot(x, w1)+b1, np.dot(x, w2)+b2), axis = 1) + return f + + def f(self, f1, f2): + f = np.concatenate((f1,f2), axis = 1) + return f + + + def get_hidden_params(self): + return self.w11, self.b11, self.w12, self.b12, self.w21, self.b21, self.w22, self.b22 + + def get_output_params(self): + return self.w1, self.b1, self.w2, self.b2 + + def get_preds(self): + return self.preds \ No newline at end of file diff --git a/LSTSVM.py b/LSTSVM.py new file mode 100755 index 0000000..0ae81ca --- /dev/null +++ b/LSTSVM.py @@ -0,0 +1,100 @@ +""" +Article : Least squares twin support vector machines for pattern classification +Link : https://sci-hub.tw/https://www.sciencedirect.com/science/article/abs/pii/S0957417408006854 +Author : Saeed Khosravi +""" +import numpy as np +class LSTSVM: + """ + Least Squares Support Vector Machines + A = Instances with label +1 + B = Instances with label -1 + C1 = hyperparameter for hyperplane 1 + C2 = hyperparameter for hyperplane 2 + + """ + def __init__(self, X, y, C1, C2, eps = 1e-4): + self.A = X[np.ix_(y[:,0] == 1),:][0,:,:] + self.B = X[np.ix_(y[:,0] == -1),:][0,:,:] + self.C1 = C1 + self.C2 = C2 + self.eps = eps + + def fit(self): + A = self.A + B = self.B + C1 = self.C1 + C2 = self.C2 + eps = self.eps + m1, n = A.shape + m2, n = B.shape + e1 = np.ones((m1, 1)) + e2 = np.ones((m2, 1)) + X = np.concatenate((A, B), axis=0) + G = np.concatenate((A, e1), axis=1) + H = np.concatenate((B, e2), axis=1) + + + if(m1 < m2): + Y = self.calc_Y_or_Z(H) + + #w1, b1 + GYGT = np.dot(np.dot(G, Y), G.T) + I = np.eye(GYGT.shape[0], GYGT.shape[1]) + w1_b1 = - np.dot(Y - np.dot(np.dot(np.dot(Y, G.T), np.linalg.inv(C1*I + GYGT)), np.dot(G, Y)), + np.dot(H.T, np.ones((H.T.shape[1], 1)))) + w1 = w1_b1[:-1, :] + b1 = w1_b1[ -1, :] + + #w2, b2 + w2_b2 = C2 * np.dot(Y - np.dot(np.dot(np.dot(Y, G.T), np.linalg.inv((I/C2)+GYGT)), np.dot(G, Y)), + np.dot(G.T, np.ones((G.T.shape[1], 1)))) + w2 = w2_b2[:-1, :] + b2 = w2_b2[ -1, :] + + else: + Z = self.calc_Y_or_Z(G) + + #w1, b1 + HZHT = np.dot(np.dot(H, Z), H.T) + I = np.eye(HZHT.shape[0], HZHT.shape[1]) + w1_b1 = -C1*np.dot(Z - np.dot(np.dot(np.dot(Z, H.T), np.linalg.inv((I/C1) + HZHT)), np.dot(H, Z)), + np.dot(H.T, np.ones((H.T.shape[1], 1)))) + w1 = w1_b1[:-1, :] + b1 = w1_b1[ -1, :] + + #w2, b2 + w2_b2 = np.dot(Z - np.dot(np.dot(np.dot(Z, H.T), np.linalg.inv(C2*I + HZHT)), np.dot(H, Z)), + np.dot(G.T, np.ones((G.T.shape[1], 1)))) + w2 = w2_b2[:-1, :] + b2 = w2_b2[ -1, :] + + self.w1 = w1 + self.w2 = w2 + self.b1 = b1 + self.b2 = b2 + + def predict(self, x_test, y_test): + distance1 = np.abs(np.dot(x_test, self.w1) + self.b1) + distance2 = np.abs(np.dot(x_test, self.w2) + self.b2) + y_pred = np.zeros_like(y_test) + for d in range(y_pred.shape[0]): + if (distance1[d] < distance2[d]): + y_pred[d][0] = 1; + else: + y_pred[d][0] = -1; + self.preds = y_pred + + def calc_Y_or_Z(self, M): + MMT = np.dot(M, M.T) + I = np.eye(MMT.shape[0], MMT.shape[1]) + tmp = np.dot(np.dot(M.T, np.linalg.inv(self.eps*I + MMT)), M) + I = np.eye(tmp.shape[0], tmp.shape[1]) + return (1/self.eps)*(I-tmp) + + def get_params(self): + return self.w1, self.b1, self.w2, self.b2 + + + def get_preds(self): + return self.preds \ No newline at end of file diff --git a/RULSTSVM.py b/RULSTSVM.py new file mode 100755 index 0000000..3b20c30 --- /dev/null +++ b/RULSTSVM.py @@ -0,0 +1,92 @@ +""" +Article: Reduced Universum Least Squares Support Vector Machine +Link : New +Author : Saeed Khosravi +""" + +import numpy as np +import math + +class RULSTSVM: + def __init__(self, X, y, C, eps): + self.X = X + self.y = y + self.C = C + self.eps = eps + + def fit(self): + self.plane1(self.X, self.y, self.C[0], self.C[1], self.C[2], self.eps) + self.plane2(self.X, self.y, self.C[3], self.C[4], self.C[5], self.eps) + + def predict(self, x_test): + distance_1 = np.abs(np.dot(x_test, self.w1) + self.b1) + distance_2 = np.abs(np.dot(x_test, self.w2) + self.b2) + y_pred = np.zeros_like(distance_1).reshape((-1, 1)) + for i in range(y_pred.shape[0]): + if (distance_1[i] < distance_2[i]): + y_pred[i][0] = 1; + else: + y_pred[i][0] = -1; + self.preds = y_pred + + def plane1(self, X, y, C1, C2, C3, eps): + S, T_, O_, e1, eg = self.definitions1(X, y) + STS = np.dot(S.T, S) + T_TT_ = np.dot(T_.T, T_) + O_TO_ = np.dot(O_.T, O_) + I = np.eye(STS.shape[0], STS.shape[1]) + v1 = -np.dot(np.linalg.inv(STS + C1*T_TT_ + C2*I + C3*O_TO_), np.dot(C1*T_.T, e1) + (1-eps)*C3*np.dot(O_.T, eg)) + self.w1 = v1[:-1, :] + self.b1 = v1[ -1, :] + + def plane2(self, X, y, C4, C5, C6, eps): + S, T, O, e1, ed = self.definitions2(X, y) + TTT = np.dot(T.T, T) + STS = np.dot(S.T, S) + OTO = np.dot(O.T, O) + I = np.dot(TTT.shape[0], TTT.shape[0]) + v2 = np.dot(np.linalg.inv(TTT + C4*STS + C5*I + C6*OTO), C4*np.dot(S.T, e1) - C6*np.dot(O.T, (1-eps)*ed)) + self.w2 = v2[:-1, :] + self.b2 = v2[ -1, :] + + def definitions1(self, X, y): + X1 = X[np.ix_(y[:,0] == 1),:][0,:,:] + X2 = X[np.ix_(y[:,0] == -1),:][0,:,:] + r, n = X1.shape + s, n = X2.shape + np.random.shuffle(X2) + X2_ = X2[:r, :] + U = X2[r: , :] + d, n = U.shape + g = math.ceil(r/2) + U_ = U[np.random.choice(np.arange(1, d), g), :] + e1 = np.ones((X1.shape[0], 1)) + eg = np.ones((U_.shape[0], 1)) + S = np.concatenate((X1 , e1), axis = 1) + T_ = np.concatenate((X2_, e1), axis = 1) + O_ = np.concatenate((U_ , eg), axis = 1) + return S, T_, O_, e1, eg + + def definitions2(self, X, y): + X1 = X[np.ix_(y[:,0] == 1),:][0,:,:] + X2 = X[np.ix_(y[:,0] == -1),:][0,:,:] + r, n = X1.shape + s, n = X2.shape + np.random.shuffle(X2) + X2_ = X2[:r, :] + U = X2[r: , :] + d, n = U.shape + g = math.ceil(r/2) + e1 = np.ones((X1.shape[0], 1)) + e2 = np.ones((X2.shape[0], 1)) + ed = np.ones((U.shape[0] , 1)) + S = np.concatenate((X1 , e1), axis = 1) + T = np.concatenate((X2 , e2), axis = 1) + O = np.concatenate((U , ed), axis = 1) + return S, T, O, e1, ed + + def get_params(self): + return self.w1, self.b1, self.w2, self.b2 + + def get_preds(self): + return self.preds \ No newline at end of file diff --git a/RUTSVM.py b/RUTSVM.py new file mode 100755 index 0000000..d8988ff --- /dev/null +++ b/RUTSVM.py @@ -0,0 +1,156 @@ +""" +Article : A reduced universum twin support vector machine for class imbalance learning +Link : https://sci-hub.tw/https://www.sciencedirect.com/science/article/abs/pii/S0031320319304510 +Author : Saeed Khosravi +""" + +import numpy as np +from cvxopt import solvers, matrix +import math + +class RUTSVM: + + def __init__(self, X, y, C1, C2, CU, eps): + + self.X = X + self.y = y + self.C1 = C1 + self.C2 = C2 + self.CU = CU + self.eps = eps + + def fit(self): + self.w1, self.b1 = self.plane1(self.X, self.y, self.C1, self.CU, self.eps) + self.w2, self.b2 = self.plane2(self.X, self.y, self.C2, self.CU, self.eps) + + def predict(self, x_test): + norm2_w1 = np.linalg.norm(self.w1) + norm2_w2 = np.linalg.norm(self.w2) + distance_1 = np.abs(np.dot(x_test, self.w1) + self.b1)/norm2_w1 + distance_2 = np.abs(np.dot(x_test, self.w2) + self.b2)/norm2_w2 + y_pred = np.zeros_like(distance_1).reshape((-1, 1)) + for i in range(y_pred.shape[0]): + if (distance_1[i] < distance_2[i]): + y_pred[i][0] = 1; + else: + y_pred[i][0] = -1; + + self.preds = y_pred + + def plane1(self, X, y, c1, cu, eps): + S, T, O, T_, O_, e1, e2, eg, ed = self.split_dataset(X, y) + m1 = S.shape[0] + m2 = T_.shape[0] + mg = O_.shape[0] + + STS = np.dot(S.T, S) + I = np.eye(STS.shape[0], STS.shape[1]) + STS_inv = np.linalg.inv(1e-4*I + STS) + + _P = np.dot(np.dot(T_, STS_inv), T_.T) + _P = np.concatenate((_P, -np.dot(np.dot(T_, STS_inv), O_.T)), axis = 1) + _P2 = -np.dot(-np.dot(O_, STS_inv), T_.T) + _P2 = np.concatenate((_P2, np.dot(np.dot(O_, STS_inv), O_.T)), axis = 1) + _P = np.concatenate((_P, _P2), axis = 0) # (m1 + mg , m1 + mg) + + _q = np.concatenate((-e1.T, (1-eps)*eg.T), axis = 1).T # (m1 + mg , 1) + + _G1 = np.concatenate(( np.eye(m1, m1), np.zeros((m1, mg))), axis = 1) + _G2 = np.concatenate((-np.eye(m1, m1), np.zeros((m1, mg))), axis = 1) + _G3 = np.concatenate(( np.zeros((mg, m1)), np.eye(mg, mg)), axis = 1) + _G4 = np.concatenate((np.zeros((mg, m1)), -np.eye(mg, mg)), axis = 1) + _G = np.concatenate((_G1, _G2), axis = 0) + _G = np.concatenate((_G , _G3), axis = 0) + _G = np.concatenate((_G, _G4), axis = 0) # (2m1 + 2mg , m1 + mg) + + _h = np.zeros((2*m1 + 2*mg, 1)) + _h[:m1, :] = c1 + _h[2*m1:2*m1+mg, :] = cu + + _P = matrix(_P, tc= 'd') + _q = matrix(_q, tc = 'd') + _G = matrix(_G, tc = 'd') + _h = matrix(_h, tc = 'd') + + qp_sol = solvers.qp(_P, _q, _G, _h, kktsolver='ldl', options={'kktreg':1e-9, 'show_progress':False}) + qp_sol = np.array(qp_sol['x']) + alphas = qp_sol[:m1, 0] + mus = qp_sol[m1:, 0] + vp = -np.dot(STS_inv, np.dot(T_.T, alphas) - np.dot(O_.T, mus)) + w = vp[:-1] + b = vp[-1] + return w, b + + + def plane2(self, X, y, c2, cu, eps): + S, T, O, T_, O_, e1, e2, eg, ed = self.split_dataset(X, y) + + m1 = S.shape[0] + m2 = T.shape[0] + md = O.shape[0] + + TTT = np.dot(T.T, T) + I = np.eye(TTT.shape[0], TTT.shape[1]) + TTT_inv = np.linalg.inv(1e-4*I + TTT) + _P = np.dot(np.dot(S, TTT_inv), S.T) + _P = np.concatenate((_P, -np.dot(np.dot(S, TTT_inv), O.T)), axis = 1) + _P2 = -np.dot(-np.dot(O, TTT_inv), S.T) + _P2 = np.concatenate((_P2, np.dot(np.dot(O, TTT_inv), O.T)), axis = 1) + _P = np.concatenate((_P, _P2), axis = 0) # (m1 + md , m1 + md) + + _q = np.concatenate((-e1.T, (eps - 1)*ed.T), axis = 1).T # (m1 + md , 1) + + _G1 = np.concatenate(( np.eye(m1, m1), np.zeros((m1, md))), axis = 1) + _G2 = np.concatenate((-np.eye(m1, m1), np.zeros((m1, md))), axis = 1) + _G3 = np.concatenate(( np.zeros((md, m1)), np.eye(md, md)), axis = 1) + _G4 = np.concatenate((np.zeros((md, m1)), -np.eye(md, md)), axis = 1) + _G = np.concatenate((_G1, _G2), axis = 0) + _G = np.concatenate((_G , _G3), axis = 0) + _G = np.concatenate((_G, _G4), axis = 0) # (2m1 + 2md , m1 + md) + + _h = np.zeros((2*m1 + 2*md, 1)) + _h[:m1, :] = c2 + _h[2*m1:2*m1+md, :] = cu + + + _P = matrix(_P, tc= 'd') + _q = matrix(_q, tc = 'd') + _G = matrix(_G, tc = 'd') + _h = matrix(_h, tc = 'd') + + qp_sol = solvers.qp(_P, _q, _G, _h, kktsolver='ldl', options={'kktreg':1e-9, 'show_progress':False}) + qp_sol = np.array(qp_sol['x']) + alphas = qp_sol[:m1, 0] + mus = qp_sol[m1:, 0] + vn = -np.dot(TTT_inv, np.dot(S.T, alphas) - np.dot(O.T, mus)) + w = vn[:-1] + b = vn[-1] + return w, b + + def split_dataset(self, X, y): + X1 = X[np.ix_(y[:,0] == 1),:][0,:,:] + X2 = X[np.ix_(y[:,0] == -1),:][0,:,:] + r, n = X1.shape + s, n = X2.shape + X2_ = X2[:r, :] + U = X2[r: , :] + d, n = U.shape + g = math.ceil(r/2) + tmp = np.random.choice(np.arange(1, d), g) + U_ = U[tmp, :] + e1 = np.ones((X1.shape[0], 1)) + e2 = np.ones((X2.shape[0], 1)) + eg = np.ones((U_.shape[0], 1)) + ed = np.ones((U.shape[0] , 1)) + S = np.concatenate((X1 , e1), axis = 1) + T_ = np.concatenate((X2_, e1), axis = 1) + T = np.concatenate((X2 , e2), axis = 1) + O_ = np.concatenate((U_ , eg), axis = 1) + O = np.concatenate((U , ed), axis = 1) + return S, T, O, T_, O_, e1, e2, eg, ed + + def get_params(self): + return self.w1, self.b1, self.w2, self.b2 + + def get_preds(self): + return self.preds \ No newline at end of file diff --git a/TSVM.py b/TSVM.py new file mode 100755 index 0000000..7959df3 --- /dev/null +++ b/TSVM.py @@ -0,0 +1,82 @@ +""" +Article : Twin Support Vector Machine +Link : https://sci-hub.tw/https://ieeexplore.ieee.org/document/4135685 +Author : Saeed Khosravi +""" + +import numpy as np +from cvxopt import solvers, matrix + +class TSVM: + + def __init__(self, X, y, C1, C2, eps=1e-4): + + self.A = X[np.ix_(y[:,0] == 1),:][0,:,:] + self.B = X[np.ix_(y[:,0] == -1),:][0,:,:] + self.C1 = C1 + self.C2 = C2 + self.eps = eps + + def fit(self): + self.w1, self.b1 = self.plane1(self.A, self.B, self.C1, self.eps) + self.w2, self.b2 = self.plane2(self.A, self.B, self.C2, self.eps) + + def predict(self, x_test): + norm2_w1 = np.linalg.norm(self.w1) + norm2_w2 = np.linalg.norm(self.w2) + distance_1 = np.abs(np.dot(x_test, self.w1) + self.b1)/norm2_w1 + distance_2 = np.abs(np.dot(x_test, self.w2) + self.b2)/norm2_w2 + y_pred = np.zeros_like(distance_1) + for i in range(y_pred.shape[0]): + if (distance_1[i] < distance_2[i]): + y_pred[i][0] = 1; + else: + y_pred[i][0] = -1; + + self.preds = y_pred + + def plane1(self, A, B, c, eps): + e1 = np.ones((A.shape[0],1)) + e2 = np.ones((B.shape[0],1)) + H = np.concatenate((A,e1), axis=1) + G = np.concatenate((B,e2), axis=1) + HTH = np.dot(H.T, H) + if np.linalg.matrix_rank(H)