#! env python import os.path import numpy import scipy.stats import rpy2.robjects as ro from rpy2.robjects import r, FloatVector, StrVector from rpy2.robjects.numpy2ri import numpy2ri import vectorize import sys ro.numpy2ri.activate() def replace(a, s, v): def f(x): if x in s: return v else: return x return numpy.vectorize(f, otypes=[object])(a) def impute(features): features = replace(features, set([vectorize.MISSING]), -1) # Dummy imputation. features = replace(features, set([vectorize.UNKNOWN]), -1) return numpy.array(features, float) def count_vals(feature): vals = [] for v in feature: if v != -1 and not v in vals: vals.append(v) return len(vals) # this function simply checks if there are enough different values # in a feature vector to apply the desired imputation strategy # if there are, the applicable strategy is returned as a string # otherwise the function returns None def validate_scale_type(feature, imp_method): if imp_method == 'skip': return None else: if '|' in imp_method: imp_methods = imp_method.split('|') else: imp_methods = [imp_method] k_vals = count_vals(feature) for imp_method in imp_methods: if imp_method == 'none' or \ ( imp_method == 'logreg' and k_vals == 2 ) or \ ( imp_method in ['polyreg', 'norm', 'norm.nob'] and k_vals > 2 ): return imp_method return None ## # mice library functions ## def merge_prev_output(features): if not os.path.isfile('features.npy'): print '[impute.py] Could not find features.npy...' return None else: print '[impute.py] Found features.npy...' merged = numpy.load('features.npy') print '[impute.py] Size of matrix in features.npy is (%i, %i)...' % (merged.shape[0], merged.shape[1]) row = 0 while row < merged.shape[0]: cur_class = merged[row][0] nxt_merged = row + 1 nxt_features = row + 1 while nxt_merged < merged.shape[0] and cur_class == merged[nxt_merged][0]: nxt_merged += 1 while nxt_features < features.shape[0] and cur_class == features[nxt_features][0]: nxt_features += 1 merged = numpy.vstack([ merged [ numpy.s_[ 0 : nxt_merged ] ], \ features[ numpy.s_[ nxt_merged : nxt_features ] ], \ merged [ numpy.s_[ nxt_merged : merged.shape[0] ] ] ]) row = nxt_features print '[impute.py] Merged matrix has size (%i, %i)...' % (merged.shape[0], merged.shape[1]) return merged # this function merges several imputed matrices returned by the R script # imputed matrices are bound together as horizontally # m1_c1 m1_c2 ... m1_cm m2_c1 m2_c2 ... m2_cm ... mk_c1 mk_c2 ... mk_cm # r1 # r2 # ... # rn def merge_imputations(features, imp_methods, k_imputations): print '[impute.py] Merging multiple imputations...' rows = features.shape[0] cols = features.shape[1] / k_imputations # for each position in the final merged matrix for r in range(rows): for c in range(cols): # make an inventory of the values from the imputed matrices vals = dict() for imp in range(k_imputations): v = features[r][c + cols * imp] if v in vals: vals[v] += 1 else: vals[v] = 1 # if the imputation strategy was for a categorical variable then # use the category that was chosen most times for imputation if imp_methods[c] in ['logreg', 'polyreg']: top_v = -1 top_k = -1 for (v, k) in vals.items(): # v is value in matrix, k is counter if top_k == -1 or top_k < k: top_k = k top_v = v features[r][c] = top_v # otherwise if the strategy was for a continuous variable then # use the average of all values used for imputation elif imp_methods[c] in ['norm', 'norm.nob']: avg = 0 for (v, k) in vals.items(): # v is value in matrix, k is counter avg += v * k avg /= float( sum(vals.values()) ) features[r][c] = avg features = numpy.delete(features, numpy.s_[cols:features.shape[1]], 1) return features # this function is simply a wrapper which calles the R function via rpy2 def impute_glue_mice(features, imp_methods, k_imputations, k_iterations): # convert features to rpy2 equivalent data type r_features = numpy2ri(features) r_imp_methods = ro.StrVector(imp_methods) # prepare the R function r.source('impute_mice.R') # read the script file r_impute = r['impute'] # get "pointer" to the function r_features = r_impute(r_features, # call R function r_imp_methods, k_imputations, k_iterations) # convert back from rpy2 format to numpy matrix shape = features.shape r_features = numpy.array(r_features).tolist() r_features = numpy.reshape(r_features, (shape[0], shape[1] * k_imputations)) # voting time features = merge_imputations(r_features, imp_methods, k_imputations) return features def impute_mice(features, imp_methods, merge = True, k_imputations = 5, k_iterations = 10): # convert MISSING and UNKNOWN to np.NAN features = replace(features, set([vectorize.MISSING]), -1) features = replace(features, set([vectorize.UNKNOWN]), -1) features = numpy.array(features, int) merge_success = False if merge: print '[impute.py] Attempting to merge feature matrix with previous output from file features.npy...' merge_res = merge_prev_output(features) if merge_res is not None: features = merge_res merge_success = True if not merge_success: # since there are features with all values missing we remove # them from the imputation procedure; save indices first idx_map = [] imputed_features = [] used_imp_methods = [] rows, cols = features.shape k_skipped = 0 for c in range(cols): val_res = validate_scale_type(features.T[c], imp_methods[c]) if val_res is not None: idx_map.append( (c, len(idx_map)) ) imputed_features.append(features.T[c]) used_imp_methods.append(val_res) else: k_skipped += 1 print '[impute.py] Can only impute %i columns ( skipped %i )' % (features.shape[1] - k_skipped - 1, k_skipped) # do imputation with mice via R imputed_features = numpy.array(imputed_features, int) imputed_features = imputed_features.T # prepare some optional args imputed_features = impute_glue_mice(imputed_features, used_imp_methods, k_imputations, k_iterations) # plug the columns back # since we have indices of columns, we transpose feature matrices features = features.T imputed_features = imputed_features.T features = features.tolist() for i1, i2 in idx_map: features[i1] = imputed_features[i2] features = numpy.array(features, int) features = features.T print '[python] Saving feature matrix of size (%i, %i) to file features.npy...' % (features.shape[0], features.shape[1]) numpy.save('features.npy', features) return features def main(): c1 = 4 c2 = 5 mat = numpy.array([ # # col1 = range(1, 11) # # col2 = 2 * col1 # # col3 = 8 * col1 # [ 1, 2, -1 ], # [ 2, 4, 16 ], # [ 3, 5, 24 ], # [ 4, 9, 32 ], # [ 5, 12, 40 ], # [ 6, 19, 48 ], # [ 7, 1, 56 ], # [ 8, 6, 64 ], # [ 9, 18, 72 ], # [10, 28, -1 ] # col1 = random # col2 = random with same sign # col3 = if col1 + col2 < 0 then c1 else c2 [ 1, -3, -3, c1], [ 1, -2, -4, c1], [ 1, -4, -2, -1], # this should be imputed to c1 [ 1, -1, -3, c1], [ 1, -4, -4, c1], [ 1, -2, -2, c1], [ 1, -3, -4, c1], [ 1, -2, -4, c1], [ 1, -4, -2, c1], [ 1, -3, -4, c1], [ 1, -3, -2, c1], [ 1, -2, -3, c1], [ 1, -3, -4, -1], [ 2, 1, 2, c2], [ 2, 3, 1, c2], [ 2, 2, 2, -1], # this should be imputed to c2 [ 2, 2, 1, c2], [ 2, 1, 2, c2], [ 2, 3, 2, c2], [ 2, 10, 5, -1], ], numpy.int) mat = impute_mice(mat, ['logreg', 'none', 'none', 'logreg']) print 'Imputed matrix is' print mat if __name__ == '__main__': main()