use std::ffi::OsString; use std::iter::IntoIterator; use std::mem::MaybeUninit; use std::sync::Arc; use std::sync::atomic::{ AtomicUsize, Ordering, }; use std::vec::Vec; use std::{ fs, io, panic, thread, }; use crossbeam_deque::{ Injector, Steal, Stealer, Worker, }; use dashmap::DashMap; use soa_rs::Soa; use crate::lexer::Token; use crate::parser::AstNode; use crate::{ Flags, err, lexer, parser, }; #[derive(Clone, Copy, Eq, Hash, PartialEq)] pub struct FileId(usize); pub struct FileData { name: Arc, buffer: Arc, tokens: Arc>>, ast: Arc>>, } impl FileData { fn new(name: OsString) -> Result { const PAD: [u8; 64] = [0; 64]; /* 512 bits */ // Append extra data to the end so that we can safely read past // instead of branching on length let mut buffer = fs::read_to_string(&name)?; buffer.push_str(unsafe { str::from_utf8_unchecked(&PAD) }); return Ok(Self { name: name.into(), buffer: buffer.into(), tokens: Arc::new_uninit(), ast: Arc::new_uninit(), }); } } pub enum Job { LexAndParse { file: FileId }, TypeCheck { file: FileId }, } pub struct CompilerState { pub files: DashMap, pub globalq: Injector, pub njobs: AtomicUsize, pub flags: Flags, } pub fn start(paths: T, flags: Flags) where T: IntoIterator, { let state = Arc::new(CompilerState { files: DashMap::new(), globalq: Injector::new(), njobs: AtomicUsize::new(0), flags, }); for (i, path) in paths.into_iter().enumerate() { let id = FileId(i); let data = match FileData::new(path.clone().into()) { Ok(x) => x, Err(e) => err!(e, "{}", path.display()), }; state.files.insert(id, data); state.njobs.fetch_add(1, Ordering::SeqCst); state.globalq.push(Job::LexAndParse { file: id }); } let mut workers = Vec::with_capacity(flags.threads); let mut stealers = Vec::with_capacity(flags.threads); for _ in 0..flags.threads { let w = Worker::new_fifo(); stealers.push(w.stealer()); workers.push(w); } let mut threads = Vec::with_capacity(flags.threads); let stealer_view: Arc<[_]> = Arc::from(stealers); for (id, w) in workers.into_iter().enumerate() { let stealer_view = Arc::clone(&stealer_view); let state = Arc::clone(&state); threads.push(thread::spawn(move || { worker_loop(id, w, stealer_view, state); })); } for t in threads { t.join().unwrap_or_else(|e| panic::resume_unwind(e)); } } fn worker_loop( id: usize, queue: Worker, stealers: Arc<[Stealer]>, state: Arc, ) { loop { if state.njobs.load(Ordering::SeqCst) == 0 { break; } let job = find_task(&queue, &state.globalq, &stealers); if let Some(job) = job { match job { Job::LexAndParse { file } => { let (name, buffer) = { let fdata = state.files.get(&file).unwrap(); (fdata.name.clone(), fdata.buffer.clone()) }; let (name, buffer) = (name.as_ref(), buffer.as_ref()); let tokens = match lexer::tokenize(name, buffer) { Ok(xs) => xs, Err(errs) => todo!(), }; let (ast, _extra_data) = parser::parse(name, &tokens); let mut fdata = state.files.get_mut(&file).unwrap(); fdata.tokens = Arc::from(MaybeUninit::new(tokens)); fdata.ast = Arc::from(MaybeUninit::new(ast)); }, _ => todo!(), } state.njobs.fetch_sub(1, Ordering::SeqCst); } else { thread::yield_now(); } } } fn find_task( localq: &Worker, globalq: &Injector, stealers: &Arc<[Stealer]>, ) -> Option { if let Some(job) = localq.pop() { return Some(job); } loop { match globalq.steal_batch_and_pop(localq) { Steal::Success(job) => return Some(job), Steal::Empty => break, Steal::Retry => continue, } } for s in stealers.iter() { loop { match s.steal_batch_and_pop(localq) { Steal::Success(job) => return Some(job), Steal::Empty => break, Steal::Retry => continue, } } } return None; }