load_data.py

import random

# data manipulation

# data  = [
#     [anser[0], anser[1], ...]
#     [same]
#     [same]
#     ...
# ]

data = [[]]
classes = []
dict_ques_title = {0: "ques about: ", 1: "ques detail: "}
dict_title = {0: "data: ", 1: "guess: ", 2: "chances: "}
dict_ques_detail = {1: "כלל לא מסכים", 2: "לא מסכים", 3: "באמצע", 4: "מסכים", 5: "מסכים מאוד", None: "None"}

# Collecting data
with open("data.csv", encoding="utf-8") as f:
    for i , line in enumerate(f.readlines()):
        if i == 0:
            line = line.replace("\"", "").replace(".", "")
            #print(line)
            start = True
            new = True
            item = ""
            for index, let in enumerate(line):
                if start and let == ",":
                    start = False
                elif not start and let == ",":
                    classes.append(item)
                    item = ""
                    new = True
                elif not start:
                    try:
                        int(let)
                        continue
                    except:
                        if new:
                            new = False
                            continue
                        item += let
            continue

        if i != 1:
            data.append([])
        line = line.replace("\"", "")
        flag = True
        item = ""
        num = 0
        for let in line:
            if num < 2 and let == "‎":
                num += 1
            elif flag and num == 2:
                flag = False
                continue
            elif num >= 2:
                if let == ",":
                    data[i - 1].append(item)
                    item = ""
                else:
                    item += let

    f.close()

dict_places = {"מושב": 0, "יישוב": 0, "עיר": 2, "קיבוץ": 1, "UN": -1, None: "None"}


# manipulation data 

def data_encode(data_s):
    data = [[]]
    for i_lst, lst in enumerate(data_s):
        if i_lst != 0:
            data.append([])
        for item in lst:
            try:
                data[i_lst].append(int(item))
            except:
                if dict_places.get(item) is None:
                    item = "UN"
                data[i_lst].append(dict_places[item])
    return data

data = data_encode(data)
def delte_nosie(data):
  for lst in data:
      if lst[0] == -1:
          data.remove(lst)
  return data

data = delte_nosie(data)
  
LABEL = 0


def find_options(data, label):
    unique = set()
    for lst in data:
        for i, item in enumerate(lst):
            if i == label:
                unique.add(item)
                break
    return unique


def decode_options(label):
    if label == 0:
        # dict = {"יישוב": 0, "עיר": 2, "קיבוץ": 1, "UN": -1, None: "None"}
        dict = {0: "יישוב/מושב", 2: "עיר", 1: "קיבוץ", -1: "UN", None: "None"}
    else:
        dict = dict_ques_detail
    return dict


dict_guess = decode_options(LABEL)


def is_numeric(value):
    """Test if a value is numeric."""
    return isinstance(value, int) or isinstance(value, float)
    
   
def split_data(data, label, test_size=0):
    if test_size < 0 or test_size > 1:
        raise SyntaxError
    if not is_numeric(test_size):
        raise SyntaxError

    x_train, x_test, y_train, y_test = [[]], [[]], [], []
    for lst in data:
        if test_size == 0:
            if len(x_train[-1]) != 0:
                x_train.append([])
            lst_x = x_train
            lst_y = y_train
        elif test_size == 1:
            if len(x_test[-1]) != 0:
                x_test.append([])
            lst_x = x_test
            lst_y = y_train
        elif len(x_test) >= len(data) / (1 / test_size):
            if len(x_train[-1]) != 0:
                x_train.append([])
            lst_x = x_train
            lst_y = y_train
        elif len(x_train) >= len(data) / (1 / (1 - test_size)):
            if len(x_test[-1]) != 0:
                x_test.append([])
            lst_x = x_test
            lst_y = y_test
        else:
            if random.choice(range(1, 11)) > test_size * 10:
                if len(x_train[-1]) != 0:
                    x_train.append([])
                lst_x = x_train
                lst_y = y_train
            else:
                if len(x_test[-1]) != 0:
                    x_test.append([])
                lst_x = x_test
                lst_y = y_test

        for i, item in enumerate(lst):
            if i == label:
                lst_y.append(item)
            else:
                lst_x[-1].append(item)

    return x_train, x_test, y_train, y_test

x_train, x_test, y_train, y_test = split_data(data, LABEL)