엉뚱뽀짝

# import time
# import concurrent.futures
#
# def worker(index) :
#     print("worker index : %s" % index)
#     time.sleep(index)
#     return ("completed %s worker job" % index)
#
# def main() :
#     future_list = []
#     executor = concurrent.futures.ProcessPoolExecutor(max_workers=3) ## executor 객체 선언. 최대 프로세스 개수 설정.
#     for i in range(5) :
#         future = executor.submit(worker, i) ## 실행할 함수와 index 등록.
#         future_list.append(future)
#     time.sleep(1)
#     for idx, future in enumerate(future_list) :
#         if future.done() : ## future가 완료되었는지 확인.
#             print("result : %s" % future.result())
#             continue
#         print("[%s worker] wait for 1 second because it has not finished yet." % idx)
#         try :
#             result = future.result(timeout=1)
#         except concurrent.futures.TimeoutError :
#             print("[%s worker] Timeout error" % idx)
#         else :
#             print("result : %s" % result)
#     executor.shutdown(wait=False)
#
# if __name__ == "__main__" :
#     main()

# worker index : 0
# worker index : 1
# worker index : 2
# worker index : 3
# result : completed 0 worker job
# [1 worker] wait for 1 second because it has not finished yet.
# worker index : 4
# result : completed 1 worker job
# [2 worker] wait for 1 second because it has not finished yet.
# [2 worker] Timeout error
# [3 worker] wait for 1 second because it has not finished yet.
# [3 worker] Timeout error
# [4 worker] wait for 1 second because it has not finished yet.
# [4 worker] Timeout error

# import time
# import concurrent.futures
#
# def worker(index) :
#     print("worker index : %s" % index)
#     time.sleep(index)
#     return ("completed %s worker job" % index)
#
# def main() :
#     with concurrent.futures.ProcessPoolExecutor(max_workers=3) as executor : ## with구문 사용으로 shutdown 생략
#         future_list = []
#         for i in range(5) :
#             future = executor.submit(worker, i) ## 실행할 함수와 index 등록.
#             future_list.append(future)
#         print(future_list)
#     finished, pending = concurrent.futures.wait(future_list, timeout =1, return_when=concurrent.futures.ALL_COMPLETED)
#     print(finished)
#     print(pending)
#     for w in finished :
#         print("finished worker : %s" % w.result())
#     for w in pending :
#         print("not finishe worker : %s" % w.result())
#
# if __name__ == "__main__" :
#     main()

# [<Future at 0x1f61da0fd30 state=running>, <Future at 0x1f61da2a9b0 state=pending>,
# <Future at 0x1f61da2aac8 state=pending>, <Future at 0x1f61da2afd0 state=pending>,
# <Future at 0x1f61da3a0b8 state=pending>]
# worker index : 0
# worker index : 1
# worker index : 2
# worker index : 3
# worker index : 4
# {<Future at 0x1f61da3a0b8 state=finished returned str>, <Future at 0x1f61da2aac8 state=finished returned str>,
# <Future at 0x1f61da0fd30 state=finished returned str>, <Future at 0x1f61da2a9b0 state=finished returned str>,
# <Future at 0x1f61da2afd0 state=finished returned str>}
# set()
# finished worker : completed 4 worker job
# finished worker : completed 2 worker job
# finished worker : completed 0 worker job
# finished worker : completed 1 worker job
# finished worker : completed 3 worker job

## 스레드에서 사용하던 Lock, RLock, Condition, Semaphore등도 multiprocessing 모듈에서 사용 가능, 구현 방법도 동일
## 그러나 일반적으로 multiprocessing 모듈에서는 사용할 필요없다. why? multiprocessing 모듈에서는 메모리 공간이 분리.
## 스레드나 프로세스와 같이 동시성을 활용한 로직 설계시 메모리 공유하지 않는게 좋다. 그러나 메모리를 공유할 경우 안전장치가 필요
## multiprocessing 모듈은 Value, Array라는 API 제공하여 메모리의 무결성 보장한다.

# import multiprocessing
#
# def worker(num, num_list) :
#     p = multiprocessing.current_process()
#     print("[%s] num : %s" % (p.name, num.value))
#     for idx, value in enumerate(num_list) :
#         print("[%s] num list[%s] : %s" % (p.name, idx, value))
#     num.value = 50
#     for i in range(len(num_list)) :
#         num_list[i] = num_list[i] * 10
#
# def main() :
#     single_integer = multiprocessing.Value("i", 5) ## i를 5로 초기화.
#     integer_list = multiprocessing.Array("i", range(10))
#     p = multiprocessing.Process(name = "worker", target = worker, args = (single_integer, integer_list))
#     p.start()
#     p.join()
#     print("num : %s" % (single_integer.value))
#     for idx, value in enumerate(integer_list) :
#         print("num list[%s] : %s" % (idx, value))
#
# if __name__ == "__main__" :
#     main()

# [worker] num : 5
# [worker] num list[0] : 0
# [worker] num list[1] : 1
# [worker] num list[2] : 2
# [worker] num list[3] : 3
# [worker] num list[4] : 4
# [worker] num list[5] : 5
# [worker] num list[6] : 6
# [worker] num list[7] : 7
# [worker] num list[8] : 8
# [worker] num list[9] : 9
# num : 50
# num list[0] : 0
# num list[1] : 10
# num list[2] : 20
# num list[3] : 30
# num list[4] : 40
# num list[5] : 50
# num list[6] : 60
# num list[7] : 70
# num list[8] : 80
# num list[9] : 90

## server process => manager API, 코드상으로는 server process는 보이지 않는다. 내부적으로 manager가 API를 담당하여 처리.

# import multiprocessing
#
# def print_array_or_list(name, values) :
#     for idx, value in enumerate(values) :
#         print("[%s] num list[%s] : %s" % (name, idx, value))
#
# def worker(v, a, l, d) :
#     p = multiprocessing.current_process()
#     print("[%s] value : %s, dict : %s" % (p.name, v, d["key"]))
#     print_array_or_list(p.name, a)
#     print_array_or_list(p.name, l)
#     v.value = 50
#     for i in range(len(a)) :
#         a[i] = a[i] * 10
#     for i in range(len(l)) :
#         l[i] = l[i] * 10
#     d["key"] = "Python3"
#
# def main() :
#     manager = multiprocessing.Manager()
#     v = manager.Value("i", 5)
#     a = manager.Array("i", range(10))
#     l = manager.list(range(10))
#     d = manager.dict()
#     d["key"] = "Python2"
#     p = multiprocessing.Process(name = "worker", target = worker, args = (v, a, l, d))
#     p.start()
#     p.join()
#     main_name = "main"
#     print("[%s] value : %s, dict : %s" % (main_name, v, d["key"]))
#     print_array_or_list(main_name, a)
#     print_array_or_list(main_name, l)
#
# if __name__ == "__main__" :
#     main()

# [worker] value : Value('i', 5), dict : Python2
# [worker] num list[0] : 0
# [worker] num list[1] : 1
# [worker] num list[2] : 2
# [worker] num list[3] : 3
# [worker] num list[4] : 4
# [worker] num list[5] : 5
# [worker] num list[6] : 6
# [worker] num list[7] : 7
# [worker] num list[8] : 8
# [worker] num list[9] : 9
# [worker] num list[0] : 0
# [worker] num list[1] : 1
# [worker] num list[2] : 2
# [worker] num list[3] : 3
# [worker] num list[4] : 4
# [worker] num list[5] : 5
# [worker] num list[6] : 6
# [worker] num list[7] : 7
# [worker] num list[8] : 8
# [worker] num list[9] : 9
# [main] value : Value('i', 50), dict : Python3
# [main] num list[0] : 0
# [main] num list[1] : 10
# [main] num list[2] : 20
# [main] num list[3] : 30
# [main] num list[4] : 40
# [main] num list[5] : 50
# [main] num list[6] : 60
# [main] num list[7] : 70
# [main] num list[8] : 80
# [main] num list[9] : 90
# [main] num list[0] : 0
# [main] num list[1] : 10
# [main] num list[2] : 20
# [main] num list[3] : 30
# [main] num list[4] : 40
# [main] num list[5] : 50
# [main] num list[6] : 60
# [main] num list[7] : 70
# [main] num list[8] : 80
# [main] num list[9] : 90
## 스레드와 달리 대부분의 API가 프로세스와 스레드에서 자원의 무결성을 보장해주므로 비교적 쉽게 사용가능.

## Process Pool
## 어떤 작업과 작업에 필요한 데이터를 pool에 등록하고 pool에서 사용할 프로세스의 개수를 입력하면 프로세스의 pool에서 작업과
## 작업에 필요한 데이트를 나눠서 정재진 프로세스만큼 작업을 나눠서 처리.

# import multiprocessing
#
# def print_initial_msg() :
#     print("start process : %s" % multiprocessing.current_process().name)
#
# def worker(data) :
#     return data * 2
#
# def main() :
#     pool = multiprocessing.Pool(processes = 4, initializer = print_initial_msg)
#     data_list = range(10)
#     result = pool.map(worker, data_list)
#     pool.close()
#     pool.join()
#     print("result : %s" % result)
#
# if __name__ == "__main__" :
#     main()

# start process : SpawnPoolWorker-3
# start process : SpawnPoolWorker-4
# start process : SpawnPoolWorker-2
# start process : SpawnPoolWorker-1
# result : [0, 2, 4, 6, 8, 10, 12, 14, 16, 18]
## 총 4개의 프로세스에서 병렬 처리된 작업의 결과가 하나로 합쳐져 출력. pool은 단순 반복 작업등을 손쉽게 처리. 빅데이터 처리에 유용.

## Coroutine
## 협력형 멀티태스킹, 이전의 스레드트 멀티프로세싱은 선점형.
## 일반적으로 함수는 한번 실행되고 종료되나 코루틴은 여러번 실행되고 여러번 종료된다.
## yield를 사용해서 여러번 반복될 수 있다.
## function, generator, coroutine.

# def coroutine() :
#     while True :
#         msg = yield ## yield를 넣어 값을 입력받을 수 있도록 한다.
#         print("hello your input message is '%s'" % msg)
#
# def main() :
#     c = coroutine()
#     next(c)
#     next(c)
#     c.send("test") ## send함수를 통해 호출.
#     c.send("coroutine")
#
# if __name__ == "__main__" :
#     main()

# hello your input message is 'None' ## yield에 아무런 값을 전달하지 않고 next 함수를 실행함.
# hello your input message is 'test'
# hello your input message is 'coroutine'

# import random
#
# TOTAL_WORK_LOAD = 50
#
# def worker() :
#     total_work_load = 0
#     worker_name = ""
#     while True :
#         worker_name, total_work_load = yield (worker_name, total_work_load) ## yield로 괄호안의 값을 입력받는다.
#         worker_load = random.randrange(1,10)
#         worker_load = worker_load if total_work_load >= worker_load else total_work_load
#         total_work_load -= worker_load
#         print("[%s] total : %s, work : %s" % (worker_name, total_work_load, worker_load)) ## yield로 받은 값을 반환
#         yield total_work_load
# ## yield로 입력과 출력을 모두 사용.
#
# def main() :
#     w1 = worker()
#     w2 = worker()
#     ret = TOTAL_WORK_LOAD
#     while ret > 0:
#         next(w1)
#         ret = w1.send(("w1", ret))
#         next(w2)
#         ret = w2.send(("w2", ret))
#
# if __name__ =="__main__" :
#     main()

## 협력형 멀티태스킹으로 하나의 작업이 끝나고 반환된 값으로 이어서 작업을 하도록 한다.
# [w1] total : 41, work : 9
# [w2] total : 36, work : 5
# [w1] total : 33, work : 3
# [w2] total : 24, work : 9
# [w1] total : 16, work : 8
# [w2] total : 10, work : 6
# [w1] total : 7, work : 3
# [w2] total : 4, work : 3
# [w1] total : 0, work : 4
# [w2] total : 0, work : 0

# def return_one_to_ten() :
#     for i in range(10) :
#         yield i
#
# def get_coroutine() :
#     yield return_one_to_ten()
#
# def main() :
#     print("== get coroutine ==")
#     c = get_coroutine()
#     print(c)
#     print("== get coroutine's return value ==")
#     ret = next(c)
#     print(ret)
#     print("== get values ==")
#     print(next(ret))
#     print(list(ret))
#
# if __name__ == "__main__" :
#     main()

# == get coroutine ==
# <generator object get_coroutine at 0x0000026D266110F8>
# == get coroutine's return value ==
# <generator object return_one_to_ten at 0x0000026D26611150>
# == get values ==
# 0
# [1, 2, 3, 4, 5, 6, 7, 8, 9]

## 앞의 예제를 yield from 구문을 추가하여 만들 경우

# def return_one_to_ten() :
#     for i in range(10) :
#         yield i
#
# def get_coroutine() :
#     yield from return_one_to_ten()
#
# def main() :
#     print("== get coroutine ==")
#     c = get_coroutine()
#     print(c)
#     print("== get values ==")
#     print(next(c))
#     print(list(c))
#
# if __name__ == "__main__" :
#     main()

# == get coroutine ==
# <generator object get_coroutine at 0x00000179871810A0>
# == get values ==
# 0
# [1, 2, 3, 4, 5, 6, 7, 8, 9]
## 실행 결과 동일하다
## yield문이 포함된 제네레이터 함수를 실행하면 제네레이터 객체가 반환되는데 이 때는 함수의 내용이 실행되지 않는다.
## next()라는 빌트인 메서드를 통해 제네레이터를 실행시킬 수 있으며 next() 메서드 내부적으로 iterator를 인자로 받아 이터레이터의
## __next__() 메서드를 실행시킨다.

## multiprocessing의 로그
# import logging
# import multiprocessing
#
# def worker(count) :
#     print("count : %s" % count)
#
# def main() :
#     multiprocessing.log_to_stderr() ## log_to_stderr()는 [%(levelname)s\%(processName)s]\ %(message)s 형식으로 로그를
## 만들어서 sys.stderr에 전달하는 역할.
#     logger = multiprocessing.get_logger()
#     logger.setLevel(logging.DEBUG)
#     for i in range(5) :
#         t = multiprocessing.Process(target = worker, args = (i,))
#         t.start()
#
# if __name__ =="__main__" :
#     main()

# [INFO/MainProcess] process shutting down
# [DEBUG/MainProcess] running all "atexit" finalizers with priority >= 0
# [INFO/MainProcess] calling join() for process Process-2
# [INFO/Process-4] child process calling self.run()
# [INFO/Process-2] child process calling self.run()
# count : 1
# [INFO/Process-2] process shutting down
# [DEBUG/Process-2] running all "atexit" finalizers with priority >= 0
# [DEBUG/Process-2] running the remaining "atexit" finalizers
# [INFO/Process-2] process exiting with exitcode 0
# count : 3
# [INFO/Process-4] process shutting down
# [DEBUG/Process-4] running all "atexit" finalizers with priority >= 0
# [DEBUG/Process-4] running the remaining "atexit" finalizers
# [INFO/Process-4] process exiting with exitcode 0
# [INFO/Process-3] child process calling self.run()
# count : 2
# [INFO/Process-3] process shutting down
# [DEBUG/Process-3] running all "atexit" finalizers with priority >= 0
# [DEBUG/Process-3] running the remaining "atexit" finalizers
# [INFO/Process-5] child process calling self.run()
# count : 4
# [INFO/Process-5] process shutting down
# [DEBUG/Process-5] running all "atexit" finalizers with priority >= 0
# [DEBUG/Process-5] running the remaining "atexit" finalizers
# [INFO/Process-5] process exiting with exitcode 0
# [INFO/Process-3] process exiting with exitcode 0
# [INFO/MainProcess] calling join() for process Process-4
# [INFO/Process-1] child process calling self.run()
# count : 0
# [INFO/Process-1] process shutting down
# [DEBUG/Process-1] running all "atexit" finalizers with priority >= 0
# [DEBUG/Process-1] running the remaining "atexit" finalizers
# [INFO/Process-1] process exiting with exitcode 0
# [INFO/MainProcess] calling join() for process Process-5
# [INFO/MainProcess] calling join() for process Process-3
# [INFO/MainProcess] calling join() for process Process-1
# [DEBUG/MainProcess] running the remaining "atexit" finalizers

## Daemon Process
## 스레드와 동일하게 프로세스도 자식 프로세스가 종료되지 않으면 메인 프로그램도 종료되지 않는다. 그래서 프로세스도 데몬으로 띄워
## 메인의 종료에 영향을 주지 않으면서 처리할 수 있다.
# import time
# import multiprocessing
#
# def daemon() :
#     print("start")
#     time.sleep(5)
#     print("exit")
#
# def main() :
#     d = multiprocessing.Process(target = daemon, name = "daemon")
#     d.daemon = True
#     d.start()
#     d.join()
#
# if __name__ == "__main__" :
#     main()

# start
# exit

## process.daemon = True로 설정할 경우, parent process가 종료될 때, child process도 같이 종료된다.
## process.daemon = False인 경우, parent process가 종료되어도 자동으로 child process가 종료되지 않는다.
## parent process는 자신의 process가 마지막으로 exit되기 전에, child process에 join을 하면서 child process의 종료를 기다린다.
## 또한 daemon으로 생성된 child process는 스스로 child process(증손자)를 가질 수 없다. 이는 parent가 종료될 때 자동으로
## daemonic child process가 종료되면서, 이 녀석이 생성한 증손자 process가 orphaned process로 되는 것을 방지하기 위함이다.
## process.join() 을 실행하면, 현재 process가 해당 process의 종료를 blocking하면서 waiting한다.

## Process Exit
## 프로세스만의 특징, 스레드의 경우 프로세스 내에서 자식으로 띄운 스레드를 종료할 수 있는 방법이 없었다. 그러나 프로세스는
## 자식을 강제로 종료시키고, 상태도 확인하고, 프로세스 수행 결과를 반환 받을 수 있다.
# import sys
# import time
# import multiprocessing
#
# def good_job() :
#     p = multiprocessing.current_process()
#     print("start name : %s, pid : %s" % (p.name, p.pid))
#     time.sleep(5)
#     print("exit name : %s, pid : %s" % (p.name, p.pid))
#     return 0
#
# def fail_job() :
#     p = multiprocessing.current_process()
#     print("start name : %s, pid : %s" % (p.name, p.pid))
#     time.sleep(5)
#     print("exit name : %s, pid : %s" % (p.name, p.pid))
#     return 0
#
# def kill_job() :
#     p = multiprocessing.current_process()
#     print("start name : %s, pid : %s" % (p.name, p.pid))
#     time.sleep(10)
#     print("exit name : %s, pid : %s" % (p.name, p.pid))
#     return 0
#
# def main() :
#     process_list = []
#     for func in [good_job, fail_job, kill_job] :
#         p = multiprocessing.Process(name = func.__name__, target = func)
#         process_list.append(p)
#         print("process check : %s, %s" % (p, p.is_alive()))
#         p.start()
#         time.sleep(0.3)
#     for p in process_list :
#         print("process check : %s, %s" % (p, p.is_alive()))
#     time.sleep(6)
#     for p in process_list :
#         print("process check : %s, %s" % (p, p.is_alive()))
#         if p.is_alive() :
#             print("termination process : %s" % p)
#             p.terminate()
#     for p in process_list :
#         print("process check : %s exit code : %s" % (p, p.exitcode))
#
# if __name__ == "__main__" :
#     main()

# process check : <Process(good_job, initial)>, False
# start name : good_job, pid : 14484
# process check : <Process(fail_job, initial)>, False
# start name : fail_job, pid : 15756
# process check : <Process(kill_job, initial)>, False
# start name : kill_job, pid : 17868
# process check : <Process(good_job, started)>, True
# process check : <Process(fail_job, started)>, True
# process check : <Process(kill_job, started)>, True
# exit name : good_job, pid : 14484
# exit name : fail_job, pid : 15756
# process check : <Process(good_job, stopped)>, False
# process check : <Process(fail_job, stopped)>, False
# process check : <Process(kill_job, started)>, True
# termination process : <Process(kill_job, started)>
# process check : <Process(good_job, stopped)> exit code : 0
# process check : <Process(fail_job, stopped)> exit code : 0
# process check : <Process(kill_job, started)> exit code : None

## Process Event
## 스레드와 마찬가지로 프로세스도 서로 통신할 수 있다. 이벤트는 멀티프로세스만의 다른 방법이 있으므로 권장되는 사항은 아니다.
# import time
# import multiprocessing
#
# def first_wait(e1, e2) :
#     p = multiprocessing.current_process()
#     while not e1.is_set() :
#         event = e1.wait(1)
#         print("[%s] event status : (%s)" % (p.name, event))
#         if event :
#             print("[%s] e1 is set" % p.name)
#             time.sleep(3)
#             print("[%s] set e2" % p.name)
#             e2.set()
#
# def second_wait(e2) :
#     p = multiprocessing.current_process()
#     while not e2.is_set() :
#         event = e2.wait(1)
#         print("[%s] event status : (%s)" % (p.name, event))
#         if event :
#             print("[%s] e2 is set" % p.name)
#
# def main() :
#     e1 = multiprocessing.Event()
#     e2 = multiprocessing.Event()
#     p1 = multiprocessing.Process(name = "first", target = first_wait, args = (e1, e2))
#     p1.start()
#     p2 = multiprocessing.Process(name = "second", target = second_wait, args = (e2,))
#     p2.start()
#     print("wait...")
#     time.sleep(5)
#     print("set e1")
#     e1.set()
#     time.sleep(5)
#     print("exit")
#
# if __name__ == "__main__" :
#     main()

# wait...
# [first] event status : (False)
# [second] event status : (False)
# [first] event status : (False)
# [second] event status : (False)
# [first] event status : (False)
# [second] event status : (False)
# [first] event status : (False)
# [second] event status : (False)
# set e1
# [first] event status : (True)
# [first] e1 is set
# [second] event status : (False)
# [second] event status : (False)
# [second] event status : (False)
# [first] set e2
# [second] event status : (True)
# [second] e2 is set
# exit

## Process Communication
## multiprocessing에는 프로세스간의 통신기능으로 queue와 pipe 기능을 지원한다.
## 프로세스간의 간단한 데이터 교환은 queue를 많이 사용.
# import time
# import multiprocessing
#
# def set_data(q) :
#     p = multiprocessing.current_process()
#     msg = "hello world"
#     q.put(msg)
#     print("[%s] set queue data : %s" % (p.name, msg))
#
# def get_data(q) :
#     time.sleep(1)
#     p = multiprocessing.current_process()
#     print("[%s] get queue data : %s" % (p.name, q.get()))
#
# def main() :
#     queue = multiprocessing.Queue()
#     p1 = multiprocessing.Process(name = "set_data", target = set_data, args = (queue,))
#     p1.start()
#     p2 = multiprocessing.Process(name = "get_data", target = get_data, args = (queue,))
#     p2.start()
#     p1.join()
#     p2.join()
#
# if __name__ == "__main__" :
#     main()

# [set_data] set queue data : hello world
# [get_data] get queue data : hello world

## pipe 방식은 기본적으로 1:1 통신 방식임. 아래의 예제로 확인
# import multiprocessing
#
# def child(pipe) :
#     p = multiprocessing.current_process()
#     msg = "hello world"
#     pipe.send(msg)
#     print("[%s] send a message to pipe %s:" % (p.name, msg))
#
# def main() :
#     parent_pipe, child_pipe = multiprocessing.Pipe()
#     p = multiprocessing.Process(name = "child", target = child, args = (child_pipe,))
#     p.start()
#     print("received message : %s" % parent_pipe.recv())
#     p.join()
#
# if __name__ == "__main__" :
#     main()

# [child] send a message to pipe hello world:
# received message : hello world