define number of threads based on available on system

This commit is contained in:
Max Mehl 2021-10-04 15:58:47 +02:00
parent 8704afc9b4
commit 551b9e3e5d
Signed by: max.mehl
GPG Key ID: 2704E4AB371E2E92
1 changed files with 2 additions and 2 deletions

View File

@ -9,7 +9,7 @@ from bs4 import BeautifulSoup
import multiprocessing.dummy as mp
import logging
from os.path import abspath
from os import environ
from os import environ, sched_getaffinity
import time
start_time = time.time()
@ -72,7 +72,7 @@ def process_file(filename: str):
)
n_processes = 4
n_processes = len(sched_getaffinity(0))
p = mp.Pool(n_processes)
logger.info("Spawning {} processes".format(n_processes))
files = list(glob.glob("**/*.xhtml", recursive=True))