9bool hila::about_to_finish =
false;
10bool hila::check_input =
false;
11int hila::check_with_nodes;
12const char *hila::input_file;
16void vector_type_info();
21int get_onoff(std::string flag)
24 if (hila::cmdline.flag_set(flag.c_str()))
26 std::string opt = hila::cmdline.get_string(flag.c_str());
27 if (opt.compare(
"on") == 0)
29 else if (opt.compare(
"off") == 0)
33 hila::out0 <<
"Command line argument " << flag <<
" requires value on/off\n";
46#if (defined(__GNUC__) && !defined(DARWIN))
56void setup_partitions();
67#if (defined(__GNUC__) && !defined(DARWIN) && !defined(_MAC_OSX_))
75 mallopt(M_TRIM_THRESHOLD, -1);
78 feenableexcept(FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW);
83 initialize_communications(argc, &argv);
98 hila::cmdline.add_flag(
"-t",
"cpu time limit");
99 hila::cmdline.add_flag(
"-o",
"output filename (default: stdout)");
100 hila::cmdline.add_flag(
"-i",
"input filename (overrides the 1st hila::input() name)\nuse '-i -' for standard input");
101 hila::cmdline.add_flag(
"-device",
"in GPU runs using only 1 GPU, choose this GPU number (default 0)");
102 hila::cmdline.add_flag(
"-check",
"check input & layout with <nodes>-nodes & exit\nonly with 1 real MPI node (don't use mpirun)");
103 hila::cmdline.add_flag(
"-n",
"number of nodes used in layout check, only relevant with -check");
104 hila::cmdline.add_flag(
"-partitions",
"number of partitioned lattice streams");
105 hila::cmdline.add_flag(
"-sync",
"synchronize partition runs (on/off) (default = off)");
110 hila::cmdline.initialise_args(argc, argv);
115 if (lattice.nodes.number == 1) {
117 if (hila::cmdline.flag_present(
"-check")) {
119 if (hila::cmdline.flag_present(
"-n")) {
120 nodes = hila::cmdline.get_int(
"-n");
125 hila::check_input =
true;
128 hila::check_with_nodes = nodes;
129 hila::out <<
"****** INPUT AND LAYOUT CHECK ******" << std::endl;
132 lattice.mynode.rank = 0;
133 lattice.nodes.number = hila::check_with_nodes;
137#if defined(CUDA) || defined(HIP)
138 if (!hila::check_input) {
140 if (hila::cmdline.flag_set(
"-device"))
141 device = hila::cmdline.get_int(
"-device");
144 hila::out0 <<
"Chose device " << device <<
"\n";
146 initialize_gpu(lattice.mynode.rank, device);
153 if (hila::partitions.number() == 1) {
156 if (hila::cmdline.flag_present(
"-o")) {
159 if (hila::cmdline.flag_set(
"-o"))
160 name = hila::cmdline.get_string(
"-o");
163 hila::out0 <<
"The name of the output file must be provided after flag '-o'!\n";
167 if (!hila::check_input) {
170 hila::out <<
"Cannot open output file " << name <<
'\n';
173 hila::out0 <<
"Output is now directed to the file '"
191 print_dashed_line(u8
"HILA ⩩ lattice framework");
192 hila::out0 <<
"Running program " << argv[0] <<
"\n";
193 hila::out0 <<
"with command line arguments '";
194 for (
int i = 1; i < argc; i++)
198#if defined(GIT_SHA_VALUE)
199#define xstr(s) makestr(s)
201 hila::out0 <<
"git SHA " << xstr(GIT_SHA_VALUE) <<
'\n';
203 hila::out0 <<
"no git information available\n";
205 hila::out0 <<
"Compiled " << __DATE__ <<
" at " << __TIME__ <<
'\n';
207 hila::out0 <<
"with options: EVEN_SITES_FIRST";
208#ifndef EVEN_SITES_FIRST
211#ifdef SPECIAL_BOUNDARY_CONDITIONS
216 hila::timestamp(
"Starting");
220 if (hila::cmdline.flag_present(
"-t")) {
222 long cputime = hila::cmdline.get_int(
"-t");
224 hila::out0 <<
"CPU time limit " << cputime <<
" seconds\n";
225 hila::setup_timelimit(cputime);
233 hila::input_file =
nullptr;
234 if (hila::cmdline.flag_present(
"-i"))
238 static const std::string input_string = hila::cmdline.get_string(
"-i");
239 hila::input_file = input_string.c_str();
240 hila::out0 <<
"Input file from command line: " << hila::input_file <<
"\n";
245 hila::out0 <<
"Using option OPENMP - with " << omp_get_max_threads() <<
" threads\n";
249#if defined(CUDA) || defined(HIP)
250 hila::out0 <<
"Using thread blocks of size " << N_threads <<
" threads\n";
252#if defined(GPU_AWARE_MPI)
258#if !defined(GPU_VECTOR_REDUCTION_THREAD_BLOCKS) || GPU_VECTOR_REDUCTION_THREAD_BLOCKS <= 0
259 hila::out0 <<
"ReductionVector with atomic operations (GPU_VECTOR_REDUCTION_THREAD_BLOCKS=0)\n";
261 hila::out0 <<
"ReductionVector with " << GPU_VECTOR_REDUCTION_THREAD_BLOCKS
262 <<
" thread blocks\n";
265 if (!hila::check_input)
275#if defined(CUDA) && !defined(PIZDAINT)
279#if (defined(__GNUC__) && !defined(DARWIN))
280 hila::out0 <<
"GNU c-library performance: not returning allocated memory\n";
290 hila::timestamp(
"Terminate");
292 hila::about_to_finish =
true;
293 if (is_comm_initialized()) {
294 finish_communications();
303void hila::error(
const char *msg) {
308void hila::error(
const std::string &msg) {
309 hila::error(msg.c_str());
322 int64_t gathers = latp->n_gather_done;
323 int64_t avoided = latp->n_gather_avoided;
325 if (gathers + avoided > 0) {
326 hila::out0 <<
" COMMS from node 0: " << gathers <<
" done, " << avoided <<
"("
327 << 100.0 * avoided / (avoided + gathers) <<
"%) optimized away\n";
329 hila::out0 <<
" No communications done from node 0\n";
334#if defined(CUDA) || defined(HIP)
338 if (hila::partitions.number() > 1) {
339 hila::timestamp(
"Waiting to sync partitions");
346 hila::timestamp(
"Finishing");
348 hila::about_to_finish =
true;
350 finish_communications();
362FILE *open_parameter_file()
364 static char parameter[] =
"parameter";
369 if (n_partitions > 1) {
370 char parameter_name[50];
372 sprintf(parameter_name,
"%s%d",parameter,this_partition);
373 fil = fopen(parameter_name,
"r");
376 fprintf(outf,
" READING PARAMETERS FROM %s\n",parameter_name);
381 fil = fopen(parameter,
"r");
383 halt(
" ** No parameter file?");
401void setup_partitions() {
404 if (hila::cmdline.flag_present(
"-partitions")) {
406 long lnum = hila::cmdline.get_int(
"-partitions");
408 hila::out0 <<
"partitions=<number> command line argument value must be positive "
409 "integer (or argument omitted)\n";
413 hila::partitions._number = lnum;
416 hila::partitions._number = 1;
418 if (hila::partitions.number() == 1)
421 hila::out0 <<
" Dividing nodes into " << hila::partitions.number() <<
" partitions\n";
425 << hila::partitions.number() <<
" partitions\n";
429#if defined(BLUEGENE_LAYOUT)
430 hila::partitions._mylattice = bg_layout_partitions(hila::partitions.number());
432 hila::partitions._mylattice =
435 if (!hila::check_input)
436 split_into_partitions(hila::partitions.mylattice());
439 if (hila::cmdline.flag_present(
"-o"))
441 std::string opt = hila::cmdline.get_string(
"-o");
442 fname = opt + std::to_string(hila::partitions.mylattice());
456 std::cout <<
"Cannot open output file " << fname <<
'\n';
467 if (!hila::check_input) {
475 << hila::partitions.number() <<
" partitions, this " << hila::partitions.mylattice()
480 if (hila::cmdline.flag_present(
"-sync"))
482 std::string onoffopt = hila::cmdline.get_string(
"-sync");
483 if (get_onoff(onoffopt) == 1)
485 hila::partitions._sync =
true;
486 hila::out0 <<
"Synchronising partition trajectories\n";
491 hila::partitions._sync =
false;
492 hila::out0 <<
"Not synchronising the partition trajectories\n"
493 <<
"Use '-sync on' command line argument to override\n";
500void vector_type_info() {
502 hila::out0 <<
"Using VCL vector class with instruction set level INSTRSET=" << INSTRSET
539 hila::out0 <<
" (You probably should use options '-mavx2 -fmad' in compilation)\n";
545void print_dashed_line(
const std::string &text) {
546 static constexpr int linelength = 60;
550 if (text.size() == 0) {
551 for (
int i = 0; i < linelength; i++)
557 for (
int i = 7 + text.size(); i < linelength; i++)
Define the logger class here.
logger_class log
Now declare the logger.
int myrank()
rank of this node
int number_of_nodes()
how many nodes there are
void synchronize()
synchronize mpi
std::ostream out
this is our default output file stream
std::ostream out0
This writes output only from main process (node 0)
std::ofstream output_file
this is just a hook to store output file, if it is in use
void initialize(int argc, char **argv)
Read in command line arguments. Initialise default stream and MPI communication.
T broadcast(T &var, int rank=0)
Broadcast the value of var to all MPI ranks from rank (default=0).
void finishrun()
Normal, controlled exit - all nodes must call this. Prints timing information and information about c...
void terminate(int status)
#define DEFAULT_OUTPUT_NAME
Default output file name.