MI_classification.py

# -*- coding: utf-8 -*-
"""
Created on Sun May 24 22:06:28 2020

@author: sudipta
"""

# -*- coding: utf-8 -*-
"""
Created on Fri May 15 18:08:29 2020

@author: sudipta
"""

import time
start_time = time.time()

from pyspark.ml.fpm import PrefixSpan
from pyspark.sql import Row
from pyspark import SparkContext
from pyspark.sql import SparkSession
from pyspark import SparkConf


spark = SparkSession \
  .builder \
  .config("spark.memory.fraction", 0.8) \
  .config("spark.executor.memory", "30g") \
  .config("spark.driver.memory", "60g")\
  .config("spark.driver.maxResultSize", "10g")\
  .config("spark.executor.heartbeatInterval", "200000")\
  .getOrCreate




f = open( "newvm100_internet_defrag_state.txt", "r")
lines = f.readlines()
catss=""
i = 44
datadefrag = [] #contains all appended stack entries of all dumps in a state
dumpinstdefrag = [] #list of all dump instances within a state
temp = []
while i < len(lines):
    
    
    if lines[i].find("Microsoft (R) Windows Debugger Version") == 0:
        temp = []
        i=i+36
         
    elif lines[i].find("Child-SP") == 0:
        catss = ""
        i=i+1
        
    elif lines[i].find("Implicit thread is now ") == 0 or lines[i].find("16.0: kd> quit") == 0:
        #this is the end of a stack
        #write the catenated substring out to a file
        #print ("catss =", catss)
        #print ("catss=", catss.lstrip())
        catss = catss.lstrip()
        datadefrag.append(catss)
        temp.append(catss)
        
        i = i + 2 
        
    elif lines[i].find ("NatVis script unloaded from ") == 0:
        i=i+22
        dumpinstdefrag.append(temp)
        
    else:
    # this is a stack entry 
        thirdss = lines[i].split(" ")[2]
        th = thirdss.strip()
        catss = catss + " " + th
        i = i + 1


f = open( "newvm100_internet_wireshark_state.txt", "r")
lines = f.readlines()
catss=""
i = 44
dataws = [] #contains all appended stack entries of all dumps in a state
dumpinstws = [] #list of all dump instances within a state
temp = []
while i < len(lines):
    
    
    if lines[i].find("Microsoft (R) Windows Debugger Version") == 0:
        temp = []
        i=i+36
         
    elif lines[i].find("Child-SP") == 0:
        catss = ""
        i=i+1
        
    elif lines[i].find("Implicit thread is now ") == 0 or lines[i].find("16.0: kd> quit") == 0:
        #this is the end of a stack
        #write the catenated substring out to a file
        #print ("catss =", catss)
        #print ("catss=", catss.lstrip())
        catss = catss.lstrip()
        dataws.append(catss)
        temp.append(catss)
        
        i = i + 2 
        
    elif lines[i].find ("NatVis script unloaded from ") == 0:
        i=i+22
        dumpinstws.append(temp)
        
    else:
    # this is a stack entry 
        thirdss = lines[i].split(" ")[2]
        th = thirdss.strip()
        catss = catss + " " + th
        i = i + 1






f = open( "newvm100_internet_procmon_state.txt", "r")
lines = f.readlines()
catss=""
i = 44
dataproc = [] #contains all appended stack entries of all dumps in a state
dumpinstproc = [] #list of all dump instances within a state
temp = []
while i < len(lines):
    
    
    if lines[i].find("Microsoft (R) Windows Debugger Version") == 0:
        temp = []
        i=i+36
         
    elif lines[i].find("Child-SP") == 0:
        catss = ""
        i=i+1
        
    elif lines[i].find("Implicit thread is now ") == 0 or lines[i].find("16.0: kd> quit") == 0:
        #this is the end of a stack
        #write the catenated substring out to a file
        #print ("catss =", catss)
        #print ("catss=", catss.lstrip())
        catss = catss.lstrip()
        dataproc.append(catss)
        temp.append(catss)
        
        i = i + 2 
        
    elif lines[i].find ("NatVis script unloaded from ") == 0:
        i=i+22
        dumpinstproc.append(temp)
        
    else:
    # this is a stack entry 
        thirdss = lines[i].split(" ")[2]
        th = thirdss.strip()
        catss = catss + " " + th
        i = i + 1
             
 
    
    
    
f = open( "newvm100.2_internet_cerber_state.txt", "r")
lines = f.readlines()
catss=""
i = 44

datacerber = [] #contains all appended stack entries of all dumps in a state
dumpinstcerber = [] #list of all dump instances within a state
temp = []
while i < len(lines):
    
    
    if lines[i].find("Microsoft (R) Windows Debugger Version") == 0:
        temp = []
        i=i+36
         
    elif lines[i].find("Child-SP") == 0:
        catss = ""
        i=i+1
        
    elif lines[i].find("Implicit thread is now ") == 0 or lines[i].find("16.0: kd> quit") == 0:
        #this is the end of a stack
        #write the catenated substring out to a file
        #print ("catss =", catss)
        #print ("catss=", catss.lstrip())
        catss = catss.lstrip()
        datacerber.append(catss)
        temp.append(catss)
        
        i = i + 2 
        
    elif lines[i].find ("NatVis script unloaded from ") == 0:
        i= i + 22
        
        dumpinstcerber.append(temp)
        
    else:
    # this is a stack entry 
        thirdss = lines[i].split(" ")[2]
        th = thirdss.strip()
        catss = catss + " " + th
        i = i + 1




f = open( "newvm100.2_internet_teslacrypt_state.txt", "r")
lines = f.readlines()
catss=""
i = 44

datatesla = [] #contains all appended stack entries of all dumps in a state
dumpinsttesla = [] #list of all dump instances within a state
temp = []
while i < len(lines):
    
    
    if lines[i].find("Microsoft (R) Windows Debugger Version") == 0:
        temp = []
        i=i+36
         
    elif lines[i].find("Child-SP") == 0:
        catss = ""
        i=i+1
        
    elif lines[i].find("Implicit thread is now ") == 0 or lines[i].find("16.0: kd> quit") == 0:
        #this is the end of a stack
        #write the catenated substring out to a file
        #print ("catss =", catss)
        #print ("catss=", catss.lstrip())
        catss = catss.lstrip()
        datatesla.append(catss)
        temp.append(catss)
        
        i = i + 2 
        
    elif lines[i].find ("NatVis script unloaded from ") == 0:
        i= i + 22
        
        dumpinsttesla.append(temp)
        
    else:
    # this is a stack entry 
        thirdss = lines[i].split(" ")[2]
        th = thirdss.strip()
        catss = catss + " " + th
        i = i + 1



f = open( "newvm100.3_internet_vipasna_state.txt", "r")
lines = f.readlines()
catss=""
i = 44

datavip = [] #contains all appended stack entries of all dumps in a state
dumpinstvip = [] #list of all dump instances within a state
temp = []
while i < len(lines):
    
    
    if lines[i].find("Microsoft (R) Windows Debugger Version") == 0:
        temp = []
        i=i+36
         
    elif lines[i].find("Child-SP") == 0:
        catss = ""
        i=i+1
        
    elif lines[i].find("Implicit thread is now ") == 0 or lines[i].find("16.0: kd> quit") == 0:
        #this is the end of a stack
        #write the catenated substring out to a file
        #print ("catss =", catss)
        #print ("catss=", catss.lstrip())
        catss = catss.lstrip()
        datavip.append(catss)
        temp.append(catss)
        
        i = i + 2 
        
    elif lines[i].find ("NatVis script unloaded from ") == 0:
        i= i + 22
        
        dumpinstvip.append(temp)
        
    else:
    # this is a stack entry 
        thirdss = lines[i].split(" ")[2]
        th = thirdss.strip()
        catss = catss + " " + th
        i = i + 1







print("done parsing")







     


import re

dataframe = []


for stack in datadefrag:
    x = len(re.findall(' +', stack))
    i = 0
    rowlist =[]
    while i <= x:
        syscall = []
        syscall.append(stack.split(" ")[i])
        rowlist.append(syscall)
        i = i+1
    dataframe.append(Row(sequence = rowlist))
        
        
sc = SparkContext.getOrCreate()        

rdd = sc.parallelize(dataframe)



spark = SparkSession(sc)
hasattr(rdd, "toDF")


prefixSpan = PrefixSpan(minSupport=round(100/len(datadefrag), 8), maxPatternLength=4,
                         maxLocalProjDBSize=32000000)  # 7/2862 for minsup

df = rdd.toDF()


prefspan= prefixSpan.findFrequentSequentialPatterns(df)



pd = prefspan.toPandas()



defragdict = {}
x = 0
for subseq in pd.sequence:
    a = subseq[0][0]
    i = 1
    while i < len(subseq):
        a = a + " " + subseq[i][0]
        i = i + 1
    defragdict[a] = pd.freq[x]
    x = x + 1

del pd





dataframe = []


for stack in dataws:
    x = len(re.findall(' +', stack))
    i = 0
    rowlist =[]
    while i <= x:
        syscall = []
        syscall.append(stack.split(" ")[i])
        rowlist.append(syscall)
        i = i+1
    dataframe.append(Row(sequence = rowlist))
        
        
sc = SparkContext.getOrCreate()        

rdd = sc.parallelize(dataframe)



spark = SparkSession(sc)
hasattr(rdd, "toDF")


prefixSpan = PrefixSpan(minSupport=round(100/len(dataws), 8), maxPatternLength=4,
                         maxLocalProjDBSize=32000000)  # 7/2833 for minsup

df = rdd.toDF()


 
prefspan= prefixSpan.findFrequentSequentialPatterns(df)



pd = prefspan.toPandas()



wsdict = {}
x = 0
for subseq in pd.sequence:
    a = subseq[0][0]
    i = 1
    while i < len(subseq):
        a = a + " " + subseq[i][0]
        i = i + 1
    wsdict[a] = pd.freq[x]
    x = x + 1
    
del pd


   
       
    
dataframe = []

for stack in dataproc:
    x = len(re.findall(' +', stack))
    i = 0
    rowlist =[]
    while i <= x:
        syscall = []
        syscall.append(stack.split(" ")[i])
        rowlist.append(syscall)
        i = i+1

    dataframe.append(Row(sequence = rowlist))
        
        
sc = SparkContext.getOrCreate()        

rdd = sc.parallelize(dataframe)



spark = SparkSession(sc)
hasattr(rdd, "toDF")


prefixSpan = PrefixSpan(minSupport=round(100/len(dataproc), 8), maxPatternLength=4,
                         maxLocalProjDBSize=32000000)  # 7/2883 for minsup

df = rdd.toDF()


 
prefspan= prefixSpan.findFrequentSequentialPatterns(df)



pd = prefspan.toPandas()



procdict = {}
x = 0
for subseq in pd.sequence:
    a = subseq[0][0]
    i = 1
    while i < len(subseq):
        a = a + " " + subseq[i][0]
        i = i + 1
    procdict[a] = pd.freq[x]
    x = x + 1
    
del pd
    
    
    
    
    
    
    
    
    
    
    
dataframe = []

for stack in datacerber:
    x = len(re.findall(' +', stack))
    i = 0
    rowlist =[]
    while i <= x:
        syscall = []
        syscall.append(stack.split(" ")[i])
        rowlist.append(syscall)
        i = i+1
    dataframe.append(Row(sequence = rowlist))
        
        
sc = SparkContext.getOrCreate()        

rdd = sc.parallelize(dataframe)



spark = SparkSession(sc)
hasattr(rdd, "toDF")


prefixSpan = PrefixSpan(minSupport=round(100/len(datacerber), 8), maxPatternLength=4,
                         maxLocalProjDBSize=32000000)  # 7/2883 for minsup

df = rdd.toDF()


 
prefspan= prefixSpan.findFrequentSequentialPatterns(df)



pd = prefspan.toPandas()



cerbdict = {}
x = 0
for subseq in pd.sequence:
    a = subseq[0][0]
    i = 1
    while i < len(subseq):
        a = a + " " + subseq[i][0]
        i = i + 1
    cerbdict[a] = pd.freq[x]
    x = x + 1
    
del pd



    
    
dataframe = []

for stack in datatesla:
    x = len(re.findall(' +', stack))
    i = 0
    rowlist =[]
    while i <= x:
        syscall = []
        syscall.append(stack.split(" ")[i])
        rowlist.append(syscall)
        i = i+1
    dataframe.append(Row(sequence = rowlist))
        
        
sc = SparkContext.getOrCreate()        

rdd = sc.parallelize(dataframe)



spark = SparkSession(sc)
hasattr(rdd, "toDF")


prefixSpan = PrefixSpan(minSupport=round(100/len(datatesla), 8), maxPatternLength=4,
                         maxLocalProjDBSize=32000000)  # 7/2883 for minsup

df = rdd.toDF()

 
prefspan= prefixSpan.findFrequentSequentialPatterns(df)



pd = prefspan.toPandas()



tesladict = {}
x = 0
for subseq in pd.sequence:
    a = subseq[0][0]
    i = 1
    while i < len(subseq):
        a = a + " " + subseq[i][0]
        i = i + 1
    tesladict[a] = pd.freq[x]
    x = x + 1    
del pd


    
    
    
dataframe = []

for stack in datavip:
    x = len(re.findall(' +', stack))
    i = 0
    rowlist =[]
    while i <= x:
        syscall = []
        syscall.append(stack.split(" ")[i])
        rowlist.append(syscall)
        i = i+1
    dataframe.append(Row(sequence = rowlist))
        
        
sc = SparkContext.getOrCreate()        

rdd = sc.parallelize(dataframe)



spark = SparkSession(sc)
hasattr(rdd, "toDF")


prefixSpan = PrefixSpan(minSupport=round(100/len(datavip), 8), maxPatternLength=4,
                         maxLocalProjDBSize=32000000)  # 7/2883 for minsup

df = rdd.toDF()


 
prefspan= prefixSpan.findFrequentSequentialPatterns(df)



pd = prefspan.toPandas()



vipdict = {}
x = 0
for subseq in pd.sequence:
    a = subseq[0][0]
    i = 1
    while i < len(subseq):
        a = a + " " + subseq[i][0]
        i = i + 1
    vipdict[a] = pd.freq[x]
    x = x + 1    
    
del pd 

    
    
vocabdict = {}


vocabdict.update(defragdict)
vocabdict.update(wsdict)
vocabdict.update(procdict)
vocabdict.update(cerbdict)
vocabdict.update(tesladict)
vocabdict.update(vipdict)


import pickle

vocabfilebvm100 = open("vocabfilebvm100.2.pkl","wb")
pickle.dump(vocabdict,vocabfilebvm100)
vocabfilebvm100.close()

print("vocabdictlen = ", len(vocabdict))






dmpdefragprefix = []


for dump in dumpinstdefrag:
    dataframe = []
    
    
    for stack in dump:
        x = len(re.findall(' +', stack))
        i = 0
        rowlist =[]
        while i <= x:
            syscall = []
            syscall.append(stack.split(" ")[i])
            rowlist.append(syscall)
            i = i+1
        dataframe.append(Row(sequence = rowlist))
            
            
    sc = SparkContext.getOrCreate()        
    
    rdd = sc.parallelize(dataframe)
    
    
    
    spark = SparkSession(sc)
    hasattr(rdd, "toDF")
    
    
    prefixSpan = PrefixSpan(minSupport= round(1/len(dump), 8) , maxPatternLength=4,
                             maxLocalProjDBSize=32000000)  # 100 for minsup or 1?
    
    df = rdd.toDF()


    
    prefspan= prefixSpan.findFrequentSequentialPatterns(df)

    
    
    pd = prefspan.toPandas()
    
    
    
    defragdict2 = {}
    x = 0
    for subseq in pd.sequence:
        a = subseq[0][0]
        i = 1
        while i < len(subseq):
            a = a + " " + subseq[i][0]
            i = i + 1
        defragdict2[a] = pd.freq[x]
        x = x + 1
    dmpdefragprefix.append(defragdict2)
    
    del pd 
    
    
print("done defrag ind PS")
    
    
    
dmpwsprefix = []


for dump in dumpinstws:
    dataframe = []
    
    
    for stack in dump:
        x = len(re.findall(' +', stack))
        i = 0
        rowlist =[]
        while i <= x:
            syscall = []
            syscall.append(stack.split(" ")[i])
            rowlist.append(syscall)
            i = i+1
        dataframe.append(Row(sequence = rowlist))
            
            
    sc = SparkContext.getOrCreate()        
    
    rdd = sc.parallelize(dataframe)
    
    
    
    spark = SparkSession(sc)
    hasattr(rdd, "toDF")
    
    
    prefixSpan = PrefixSpan(minSupport= round(1/len(dump), 8) , maxPatternLength=4,
                             maxLocalProjDBSize=32000000)  # 100 for minsup or 1?
    
    df = rdd.toDF()


     
    prefspan= prefixSpan.findFrequentSequentialPatterns(df)

    
    
    pd = prefspan.toPandas()
    
    
    
    wsdict2 = {}
    x = 0
    for subseq in pd.sequence:
        a = subseq[0][0]
        i = 1
        while i < len(subseq):
            a = a + " " + subseq[i][0]
            i = i + 1
        wsdict2[a] = pd.freq[x]
        x = x + 1
    dmpwsprefix.append(wsdict2)
    
    del pd 
    
print("done ws ind PS")
   



 
    
dmpcerbprefix = []


for dump in dumpinstcerber:
    dataframe = []
    
    
    for stack in dump:
        x = len(re.findall(' +', stack))
        i = 0
        rowlist =[]
        while i <= x:
            syscall = []
            syscall.append(stack.split(" ")[i])
            rowlist.append(syscall)
            i = i+1
        dataframe.append(Row(sequence = rowlist))
            
            
    sc = SparkContext.getOrCreate()        
    
    rdd = sc.parallelize(dataframe)
    
    
    
    spark = SparkSession(sc)
    hasattr(rdd, "toDF")
    
    
    prefixSpan = PrefixSpan(minSupport= round(1/len(dump), 8) , maxPatternLength=4,
                             maxLocalProjDBSize=32000000)  # 100 for minsup or 1?
    
    df = rdd.toDF()

     
    prefspan= prefixSpan.findFrequentSequentialPatterns(df)

    
    
    pd = prefspan.toPandas()
    
    
    
    cerbdict2 = {}
    x = 0
    for subseq in pd.sequence:
        a = subseq[0][0]
        i = 1
        while i < len(subseq):
            a = a + " " + subseq[i][0]
            i = i + 1
        cerbdict2[a] = pd.freq[x]
        x = x + 1
    dmpcerbprefix.append(cerbdict2)
    
    del pd 

print("done cerber ind PS")

    
    
dmpprocprefix = []


for dump in dumpinstproc:
    dataframe = []
    
    
    for stack in dump:
        x = len(re.findall(' +', stack))
        i = 0
        rowlist =[]
        while i <= x:
            syscall = []
            syscall.append(stack.split(" ")[i])
            rowlist.append(syscall)
            i = i+1
        dataframe.append(Row(sequence = rowlist))
            
            
    sc = SparkContext.getOrCreate()        
    
    rdd = sc.parallelize(dataframe)
    
    
    
    spark = SparkSession(sc)
    hasattr(rdd, "toDF")
    
    
    prefixSpan = PrefixSpan(minSupport= round(1/len(dump), 8) , maxPatternLength=4,
                             maxLocalProjDBSize=32000000)  # 100 for minsup or 1?
    
    df = rdd.toDF()

     
    prefspan= prefixSpan.findFrequentSequentialPatterns(df)

    
    
    pd = prefspan.toPandas()
    
    
    
    procdict2 = {}
    x = 0
    for subseq in pd.sequence:
        a = subseq[0][0]
        i = 1
        while i < len(subseq):
            a = a + " " + subseq[i][0]
            i = i + 1
        procdict2[a] = pd.freq[x]
        x = x + 1
    dmpprocprefix.append(procdict2)
    
    del pd 
    
print("done proc ind PS")





dmpteslaprefix = []


for dump in dumpinsttesla:
    dataframe = []
    
    
    for stack in dump:
        x = len(re.findall(' +', stack))
        i = 0
        rowlist =[]
        while i <= x:
            syscall = []
            syscall.append(stack.split(" ")[i])
            rowlist.append(syscall)
            i = i+1
        dataframe.append(Row(sequence = rowlist))
            
            
    sc = SparkContext.getOrCreate()        
    
    rdd = sc.parallelize(dataframe)
    
    
    
    spark = SparkSession(sc)
    hasattr(rdd, "toDF")
    
    
    prefixSpan = PrefixSpan(minSupport= round(1/len(dump), 8) , maxPatternLength=4,
                             maxLocalProjDBSize=32000000)  # 100 for minsup or 1?
    
    df = rdd.toDF()
     
    prefspan= prefixSpan.findFrequentSequentialPatterns(df)

    
    
    pd = prefspan.toPandas()
    
    
    
    tesladict2 = {}
    x = 0
    for subseq in pd.sequence:
        a = subseq[0][0]
        i = 1
        while i < len(subseq):
            a = a + " " + subseq[i][0]
            i = i + 1
        tesladict2[a] = pd.freq[x]
        x = x + 1
    dmpteslaprefix.append(tesladict2)


    del pd 

print("done tesla ind PS")



dmpvipprefix = []


for dump in dumpinstvip:
    dataframe = []
    
    
    for stack in dump:
        x = len(re.findall(' +', stack))
        i = 0
        rowlist =[]
        while i <= x:
            syscall = []
            syscall.append(stack.split(" ")[i])
            rowlist.append(syscall)
            i = i+1
        dataframe.append(Row(sequence = rowlist))
            
            
    sc = SparkContext.getOrCreate()        
    
    rdd = sc.parallelize(dataframe)
    
    
    
    spark = SparkSession(sc)
    hasattr(rdd, "toDF")
    
    
    prefixSpan = PrefixSpan(minSupport= round(1/len(dump), 8) , maxPatternLength=4,
                             maxLocalProjDBSize=32000000)  # 100 for minsup or 1?
    
    df = rdd.toDF()

     
    prefspan= prefixSpan.findFrequentSequentialPatterns(df)

    
    
    pd = prefspan.toPandas()
    
    
    
    vipdict2 = {}
    x = 0
    for subseq in pd.sequence:
        a = subseq[0][0]
        i = 1
        while i < len(subseq):
            a = a + " " + subseq[i][0]
            i = i + 1
        vipdict2[a] = pd.freq[x]
        x = x + 1
    dmpvipprefix.append(vipdict2)
    
    del pd 









listdefragcount =[]
i =0
while i<=99:
    
    countdictdefrag ={} 
    
    for subseq in vocabdict.keys():
        binary = 0
        countdictdefrag[subseq] = binary
        for key, value in dmpdefragprefix[i].items():
            if (subseq == key):
                binary = 1
                countdictdefrag[subseq] = binary
            else:
                pass
                             
    listdefragcount.append(countdictdefrag)
    i = i + 1





    

listwscount =[]
i =0
while i<=99:
    
    countdictws ={} 
    
    for subseq in vocabdict.keys():
        binary = 0
        countdictws[subseq] = binary
        for key, value in dmpwsprefix[i].items():
            if (subseq == key):
                binary = 1
                countdictws[subseq] = binary
            else:
                pass
                             
    listwscount.append(countdictws)
    i = i + 1





listproccount =[]
i =0
while i<=99:
    
    countdictproc ={} 
    
    for subseq in vocabdict.keys():
        binary = 0
        countdictproc[subseq] = binary
        for key, value in dmpprocprefix[i].items():
            if (subseq == key):
                binary = 1
                countdictproc[subseq] = binary
            else:
                pass
                             
    listproccount.append(countdictproc)
    i = i + 1






listcerbcount =[]
i =0
while i<=99:
    
    countdictcerb ={} 
    
    for subseq in vocabdict.keys():
        binary = 0
        countdictcerb[subseq] = binary
        for key, value in dmpcerbprefix[i].items():
            if (subseq == key):
                binary = 1
                countdictcerb[subseq] = binary
            else:
                pass
                             
    listcerbcount.append(countdictcerb)
    i = i + 1




listteslacount =[]
i =0
while i<=99:
    
    countdicttesla ={} 
    
    for subseq in vocabdict.keys():
        binary = 0
        countdicttesla[subseq] = binary
        for key, value in dmpteslaprefix[i].items():
            if (subseq == key):
                binary = 1
                countdicttesla[subseq] = binary
            else:
                pass
                             
    listteslacount.append(countdicttesla)
    i = i + 1



listvipcount =[]
i =0
while i<=99:
    
    countdictvip ={} 
    
    for subseq in vocabdict.keys():
        binary = 0
        countdictvip[subseq] = binary
        for key, value in dmpvipprefix[i].items():
            if (subseq == key):
                binary = 1
                countdictvip[subseq] = binary
            else:
                pass
                             
    listvipcount.append(countdictvip)
    i = i + 1







 
    
import numpy as numpy
newlist =[]

for countdictdefrag in listdefragcount: 
    list1 = list (countdictdefrag.values())
    outarray1 = numpy.asarray(list1)
    newlist.append(outarray1)
    
for countdictws in listwscount:    
    list2 = list (countdictws.values())
    outarray2 = numpy.asarray(list2)
    newlist.append(outarray2)    
    
    
for countdictproc in listproccount:    
    list3 = list (countdictproc.values())
    outarray3 = numpy.asarray(list3)
    newlist.append(outarray3)

for countdictcerb in listcerbcount:    
    list4 = list (countdictcerb.values())
    outarray4 = numpy.asarray(list4)
    newlist.append(outarray4)


for countdicttesla in listteslacount:    
    list5 = list (countdicttesla.values())
    outarray5 = numpy.asarray(list5)
    newlist.append(outarray5)


for countdictvip in listvipcount:    
    list6 = list (countdictvip.values())
    outarray6 = numpy.asarray(list6)
    newlist.append(outarray6)
   
    
    
finalarray = numpy.asarray(newlist)   



a = numpy.zeros((300,), dtype=int)
b = numpy.ones((300,), dtype=int)

targetarray = numpy.concatenate((a, b), axis=None)



X = finalarray

y = targetarray

import pickle

finalarraybvm100 = open("finalarraybvm100.2bin.pkl","wb")
pickle.dump(finalarray,finalarraybvm100)
finalarraybvm100.close()


from sklearn.model_selection import train_test_split   
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)




from sklearn.ensemble import RandomForestClassifier

clf=RandomForestClassifier(n_estimators=100)   
clf.fit(X_train,y_train)

y_pred = clf.predict(X_test)

from sklearn import metrics
print("Accuracy:",metrics.accuracy_score(y_test, y_pred))



import pickle
with open('model_pickle100.2bin', 'wb') as f:
    pickle.dump(clf, f)



print("--- %s seconds ---" % (time.time() - start_time))