PCA

yf

关键句子

import numpy as np
Mat = np.array(mat, dtype='float64')
p,n = np.shape(Mat) # shape of Mat 
t = np.mean(Mat, 0) # mean of each column

# substract the mean of each column
for i in range(p):
    for j in range(n):
        Mat[i,j] = float(Mat[i,j]-t[j])

# covariance Matrix
cov_Mat = np.dot(Mat.T, Mat)/(p-1
# method 1: PCA by original algorithm
# eigvalues and eigenvectors of covariance Matrix with eigvalues descending
U,V = np.linalg.eigh(cov_Mat) 
# Rearrange the eigenvectors and eigenvalues
U = U[::-1]
for i in range(n):
    V[i,:] = V[i,:][::-1]
# choose eigenvalue by component or rate, not both of them euqal to 0
Index = 2  # choose how many main factors
v = V[:,:Index]  # subset of Unitary matrix
T1 = np.dot(Mat, v)
print('We choose %d main factors.'%Index)
print('After PCA transformation, data becomes:\n',T1)

全部

import numpy as np
from sklearn.decomposition import PCA
import sys
#returns choosing how many main factors
def index_lst(lst, component=0, rate=0):
    #component: numbers of main factors
    #rate: rate of sum(main factors)/sum(all factors)
    #rate range suggest: (0.8,1)
    #if you choose rate parameter, return index = 0 or less than len(lst)
    if component and rate:
        print('Component and rate must choose only one!')
        sys.exit(0)
    if not component and not rate:
        print('Invalid parameter for numbers of components!')
        sys.exit(0)
    elif component:
        print('Choosing by component, components are %s......'%component)
        return component
    else:
        print('Choosing by rate, rate is %s ......'%rate)
        for i in range(1, len(lst)):
            if sum(lst[:i])/sum(lst) >= rate:
                return i
        return 0

def main():
    # test data
    mat = [[-1,-1,0,2,1],[2,0,0,-1,-1],[2,0,1,1,0]]

    # simple transform of test data
    Mat = np.array(mat, dtype='float64')
    print('Before PCA transforMation, data is:\n', Mat)
    print('\nMethod 1: PCA by original algorithm:')
    p,n = np.shape(Mat) # shape of Mat # n是feature个数
    t = np.mean(Mat, 0) # mean of each column

    # substract the mean of each column
    for i in range(p):
        for j in range(n):
            Mat[i,j] = float(Mat[i,j]-t[j])

    # covariance Matrix
    cov_Mat = np.dot(Mat.T, Mat)/(p-1)

    # method 1: PCA by original algorithm
    # eigvalues and eigenvectors of covariance Matrix with eigvalues descending
    U,V = np.linalg.eigh(cov_Mat) 
    # Rearrange the eigenvectors and eigenvalues
    U = U[::-1]
    for i in range(n):
        V[i,:] = V[i,:][::-1]
    # choose eigenvalue by component or rate, not both of them euqal to 0
    Index = index_lst(U, component=2)  # choose how many main factors
    print(f'locate 1, index={Index}')
    if Index:
        v = V[:,:Index]  # subset of Unitary matrix
    else:  # improper rate choice may return Index=0
        print('Invalid rate choice.\nPlease adjust the rate.')
        print('Rate distribute follows:')
        print([sum(U[:i])/sum(U) for i in range(1, len(U)+1)])
        sys.exit(0)
    # data transformation
    T1 = np.dot(Mat, v)
    # print the transformed data
    print('We choose %d main factors.'%Index)
    print('After PCA transformation, data becomes:\n',T1)

    # method 2: PCA by original algorithm using SVD
    print('\nMethod 2: PCA by original algorithm using SVD:')
    # u: Unitary matrix,  eigenvectors in columns 
    # d: list of the singular values, sorted in descending order
    u,d,v = np.linalg.svd(cov_Mat)
    Index = index_lst(d, rate=0.95)  # choose how many main factors
    T2 = np.dot(Mat, u[:,:Index])  # transformed data
    print('We choose %d main factors.'%Index)
    print('After PCA transformation, data becomes:\n',T2)

    # method 3: PCA by Scikit-learn
    pca = PCA(n_components=2) # n_components can be integer or float in (0,1)
    pca.fit(mat)  # fit the model
    print('\nMethod 3: PCA by Scikit-learn:')
    print('After PCA transformation, data becomes:')
    print(pca.fit_transform(mat))  # transformed data

main()

问题1:为什么计算协方差的时候，分母是p-1 而不是p？

一句话解释：因为均值已经用了 p个数的平均来求方差，只有 p-1个数和均值的计算是不相关的。也就是说 第n个数可以用前面的 n-1个数和均值来唯一确定，实际上没有信息量，所以只除以 n-1

问题2:怎么计算协方差（covariance）？

问题3: 和 svd 什么关系？

矫正之后

import numpy as np
from sklearn.decomposition import PCA
import sys
#returns choosing how many main factors
def index_lst(lst, component=0, rate=0):
    #component: numbers of main factors
    #rate: rate of sum(main factors)/sum(all factors)
    #rate range suggest: (0.8,1)
    #if you choose rate parameter, return index = 0 or less than len(lst)
    if component and rate:
        print('Component and rate must choose only one!')
        sys.exit(0)
    if not component and not rate:
        print('Invalid parameter for numbers of components!')
        sys.exit(0)
    elif component:
        print('Choosing by component, components are %s......'%component)
        return component
    else:
        print('Choosing by rate, rate is %s ......'%rate)
        for i in range(1, len(lst)):
            if sum(lst[:i])/sum(lst) >= rate:
                return i
        return 0


def main():
    # test data
    mat = [[-1,-1,0,2,1],[2,0,0,-1,-1],[2,0,1,1,0]]

    # simple transform of test data
    Mat = np.array(mat, dtype='float64')
    print('Before PCA transforMation, data is:\n', Mat)
    print('\nMethod 1: PCA by original algorithm:')
    p,n = np.shape(Mat) # shape of Mat 
    X_mean_centered = np.mean(Mat, axis=0) # mean of each column
    Mat = Mat - X_mean_centered

    # covariance Matrix
    cov_Mat = np.dot(Mat.T, Mat)/(p-1)

    # method 1: PCA by original algorithm
    # eigvalues and eigenvectors of covariance Matrix with eigvalues descending
    eigenvalues, eign_V = np.linalg.eigh(cov_Mat) 
    # Rearrange the eigenvectors and eigenvalues
    eigenvalues = eigenvalues[::-1]
    for i in range(n):
        eign_V[i,:] = eign_V[i,:][::-1]
    # choose eigenvalue by component or rate, not both of them euqal to 0
    Index = index_lst(eigenvalues, component=2)  # choose how many main factors
    if Index:
        v = eign_V[:,:Index]  # subset of Unitary matrix
    else:  # improper rate choice may return Index=0
        print('Invalid rate choice.\nPlease adjust the rate.')
        print('Rate distribute follows:')
        print([sum(eigenvalues[:i])/sum(eigenvalues) for i in range(1, len(eigenvalues)+1)])
        sys.exit(0)
    # data transformation
    T1 = np.dot(Mat, v)
    # print the transformed data
    print('We choose %d main factors.'%Index)
    print('After PCA transformation, data becomes:\n',T1)

    # method 2: PCA SVD
    print('\nMethod 2: PCA by original algorithm using SVD:')
    # u: Unitary matrix,  eigenvectors in columns 
    # d: list of the singular values, sorted in descending order
    u,d,v = np.linalg.svd(Mat) # u 的每一列是 Mat*Mat.T的特征向量，v的每一行是 Mat.T * Mat的特征向量。
    # svd_flip， https://github.com/scikit-learn/scikit-learn/blob/main/sklearn/utils/extmath.py
    signs = np.sign(u[np.argmax(np.abs(u), axis=0), range(u.shape[1])]) # 每个行索引是取的列的最大值的索引，列索引不变。
    print('signs={signs}')
    u *= signs[np.newaxis, :]
    v[:len(signs), :] *= signs[:, np.newaxis]

    Index = 2  # choose how many main factors
    T2 = np.dot(Mat, v.T[:, :Index]) # transformed data
    print('We choose %d main factors.'%Index)
    print('After PCA transformation, data becomes:\n',T2)

    # method 3: PCA by Scikit-learn
    pca = PCA(n_components=2) # n_components can be integer or float in (0,1)
    pca.fit(mat)  # fit the model
    print('\nMethod 3: PCA by Scikit-learn:')
    print('After PCA transformation, data becomes:')
    print(pca.fit_transform(mat))  # transformed data

main()

https://zhuanlan.zhihu.com/p/29846048

https://www.cnblogs.com/lpzblog/p/9519756.html

https://www.youtube.com/watch?v=Rjr62b_h7S4&list=PLcWfeUsAys2k_xub3mHks85sBHZvg24Jd&index=8