Module OpenMPI
The option #SBATCH --nodes N is only needed when you want to explicitly control the number of nodes. Typically, you should not specify the number of nodes but instead use #SBATCH --tasks N to let MPI run N processes on any number of nodes.
Single-node MPI job with no threads
Create the MPI code.
mpi_singlenode_nothreads.c
#include <stdio.h>
#include <mpi.h>
int main(int argc, char** argv) {
int process_rank, number_processes, hostname_length;
char hostname[MPI_MAX_PROCESSOR_NAME];
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &process_rank);
MPI_Comm_size(MPI_COMM_WORLD, &number_processes);
MPI_Get_processor_name(hostname, &hostname_length);
printf("Hello World from process %d of %d. Running on node %s\n", process_rank, number_processes, hostname);
MPI_Finalize();
return 0;
}
Load OpenMPI module.
module load mpi/openmpi-4.1.6
Compile the MPI code.
mpicc -o mpi_singlenode_nothreads mpi_singlenode_nothreads.c
Create the Slurm script file. Adjust the number of processes (--ntasks 4) and the amount of memory (--mem 4G) needed for your job.
mpi_singlenode_nothreads_slurm_script.sh
#!/bin/bash
#SBATCH --job-name MPI_singlenode_nothreads
#SBATCH --output MPI_singlenode_nothreads_output.log
#SBATCH --error MPI_singlenode_nothreads_error.log
#SBATCH --nodes 1
#SBATCH --ntasks 4
#SBATCH --mem 4G
module load mpi/openmpi-4.1.6
srun mpi_singlenode_nothreads
Submit the job to Slurm.
sbatch mpi_singlenode_nothreads_slurm_script.shMonitor the execution of your job in the Slurm queue.
squeue -u $USERThe job may show up in the queue as running (R) or pending (PD). When the job completes, it will disappear from the queue and you can check the output.
[user@athena1 ~]$ module load mpi/openmpi-4.1.6
[user@athena1 ~]$ mpicc -o mpi_singlenode_nothreads mpi_singlenode_nothreads.c
[user@athena1 ~]$ sbatch mpi_singlenode_nothreads_slurm_script.sh
Submitted batch job 123
[user@athena1 ~]$ squeue -u $USER
JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON) ## Job is running with JOBID 123
123 cpu MPI_singlenode_nothreads user R 0:01 1 athena101
[user@athena1 ~]$ squeue -u $USER
JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON) ## JOBID 123 completed and exited the queue
[user@athena1 ~]$ cat MPI_singlenode_nothreads_output.log
Hello World from process 0 of 4. Running on node athena101
Hello World from process 1 of 4. Running on node athena101
Hello World from process 2 of 4. Running on node athena101
Hello World from process 3 of 4. Running on node athena101
Single-node MPI job with OpenMP threads
Create the MPI code.
mpi_singlenode_ompthreads.c
#include <stdio.h>
#include <mpi.h>
#include <omp.h>
int main(int argc, char** argv) {
int process_rank, number_processes, hostname_length;
char hostname[MPI_MAX_PROCESSOR_NAME];
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &process_rank);
MPI_Comm_size(MPI_COMM_WORLD, &number_processes);
MPI_Get_processor_name(hostname, &hostname_length);
#pragma omp parallel
printf("Hello World from process %d of %d. Thread %d of %d. Running on node %s\n", process_rank, number_processes, omp_get_thread_num(), omp_get_num_threads(), hostname);
MPI_Finalize();
return 0;
}
Load OpenMPI module.
module load mpi/openmpi-4.1.6
Compile the MPI code with OpenMP.
mpicc -fopenmp -o mpi_singlenode_ompthreads mpi_singlenode_ompthreads.c
Create the Slurm script file. Adjust the number of processes (--ntasks 4), the number of threads per process (--cpus-per-task 3) and the amount of memory (--mem 4G) needed for your job.
mpi_singlenode_ompthreads_slurm_script.sh
#!/bin/bash
#SBATCH --job-name MPI_singlenode_ompthreads
#SBATCH --output MPI_singlenode_ompthreads_output.log
#SBATCH --error MPI_singlenode_ompthreads_error.log
#SBATCH --nodes 1
#SBATCH --ntasks 4
#SBATCH --cpus-per-task 3
#SBATCH --mem 4G
module load mpi/openmpi-4.1.6
export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK
srun mpi_singlenode_ompthreads
Submit the job to Slurm.
sbatch mpi_singlenode_ompthreads_slurm_script.shMonitor the execution of your job in the Slurm queue.
squeue -u $USERThe job may show up in the queue as running (R) or pending (PD). When the job completes, it will disappear from the queue and you can check the output.
[user@athena1 ~]$ module load mpi/openmpi-4.1.6
[user@athena1 ~]$ mpicc -fopenmp -o mpi_singlenode_ompthreads mpi_singlenode_ompthreads.c
[user@athena1 ~]$ sbatch mpi_singlenode_ompthreads_slurm_script.sh
Submitted batch job 123
[user@athena1 ~]$ squeue -u $USER
JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON) ## Job is running with JOBID 123
123 cpu MPI_singlenode_ompthreads user R 0:01 1 athena101
[user@athena1 ~]$ squeue -u $USER
JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON) ## JOBID 123 completed and exited the queue
[user@athena1 ~]$ cat MPI_singlenode_ompthreads_output.log
Hello World from process 0 of 4. Thread 0 of 3. Running on node athena101
Hello World from process 0 of 4. Thread 1 of 3. Running on node athena101
Hello World from process 0 of 4. Thread 2 of 3. Running on node athena101
Hello World from process 1 of 4. Thread 0 of 3. Running on node athena101
Hello World from process 1 of 4. Thread 1 of 3. Running on node athena101
Hello World from process 1 of 4. Thread 2 of 3. Running on node athena101
Hello World from process 2 of 4. Thread 0 of 3. Running on node athena101
Hello World from process 2 of 4. Thread 1 of 3. Running on node athena101
Hello World from process 2 of 4. Thread 2 of 3. Running on node athena101
Hello World from process 3 of 4. Thread 0 of 3. Running on node athena101
Hello World from process 3 of 4. Thread 1 of 3. Running on node athena101
Hello World from process 3 of 4. Thread 2 of 3. Running on node athena101
Multi-node MPI job with no threads
Create the MPI code.
mpi_multinode_nothreads.c
#include <stdio.h>
#include <mpi.h>
int main(int argc, char** argv) {
int process_rank, number_processes, hostname_length;
char hostname[MPI_MAX_PROCESSOR_NAME];
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &process_rank);
MPI_Comm_size(MPI_COMM_WORLD, &number_processes);
MPI_Get_processor_name(hostname, &hostname_length);
printf("Hello World from process %d of %d. Running on node %s\n", process_rank, number_processes, hostname);
MPI_Finalize();
return 0;
}
Load OpenMPI module.
module load mpi/openmpi-4.1.6
Compile the MPI code.
mpicc -o mpi_multinode_nothreads mpi_multinode_nothreads.c
Create the Slurm script file. Adjust the number of nodes (--nodes 2), the number of processes (--ntasks 4) and the amount of memory (--mem 4G) needed for your job.
mpi_multinode_nothreads_slurm_script.sh
#!/bin/bash
#SBATCH --job-name MPI_multinode_nothreads
#SBATCH --output MPI_multinode_nothreads_output.log
#SBATCH --error MPI_multinode_nothreads_error.log
#SBATCH --nodes 2
#SBATCH --ntasks 4
#SBATCH --mem 4G
module load mpi/openmpi-4.1.6
srun mpi_multinode_nothreads
Submit the job to Slurm.
sbatch mpi_multinode_nothreads_slurm_script.shMonitor the execution of your job in the Slurm queue.
squeue -u $USERThe job may show up in the queue as running (R) or pending (PD). When the job completes, it will disappear from the queue and you can check the output.
[user@athena1 ~]$ module load mpi/openmpi-4.1.6
[user@athena1 ~]$ mpicc -o mpi_multinode_nothreads mpi_multonode_nothreads.c
[user@athena1 ~]$ sbatch mpi_multinode_nothreads_slurm_script.sh
Submitted batch job 123
[user@athena1 ~]$ squeue -u $USER
JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON) ## Job is running with JOBID 123
123 cpu MPI_multinode_nothreads user R 0:01 2 athena[101,201]
[user@athena1 ~]$ squeue -u $USER
JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON) ## JOBID 123 completed and exited the queue
[user@athena1 ~]$ cat MPI_multinode_nothreads_output.log
Hello World from process 0 of 4. Running on node athena101
Hello World from process 1 of 4. Running on node athena101
Hello World from process 2 of 4. Running on node athena201
Hello World from process 3 of 4. Running on node athena201
Multi-node MPI job with OpenMP threads
Create the MPI code.
mpi_multinode_ompthreads.c
#include <stdio.h>
#include <mpi.h>
#include <omp.h>
int main(int argc, char** argv) {
int process_rank, number_processes, hostname_length;
char hostname[MPI_MAX_PROCESSOR_NAME];
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &process_rank);
MPI_Comm_size(MPI_COMM_WORLD, &number_processes);
MPI_Get_processor_name(hostname, &hostname_length);
#pragma omp parallel
printf("Hello World from process %d of %d. Thread %d of %d. Running on node %s\n", process_rank, number_processes, omp_get_thread_num(), omp_get_num_threads(), hostname);
MPI_Finalize();
return 0;
}
Load OpenMPI module.
module load mpi/openmpi-4.1.6
Compile the MPI code with OpenMP.
mpicc -fopenmp -o mpi_multinode_ompthreads mpi_multinode_ompthreads.c
Create the Slurm script file. Adjust the number of nodes (--nodes 2), the number of processes (--ntasks 4), the number of threads per process (--cpus-per-task 3) and the amount of memory (--mem 4G) needed for your job.
mpi_multinode_ompthreads_slurm_script.sh
#!/bin/bash
#SBATCH --job-name MPI_multinode_ompthreads
#SBATCH --output MPI_multinode_ompthreads_output.log
#SBATCH --error MPI_multinode_ompthreads_error.log
#SBATCH --nodes 2
#SBATCH --ntasks 4
#SBATCH --cpus-per-task 3
#SBATCH --mem 4G
module load mpi/openmpi-4.1.6
export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK
srun mpi_multinode_ompthreads
Submit the job to Slurm.
sbatch mpi_multinode_ompthreads_slurm_script.shMonitor the execution of your job in the Slurm queue.
squeue -u $USERThe job may show up in the queue as running (R) or pending (PD). When the job completes, it will disappear from the queue and you can check the output.
[user@athena1 ~]$ module load mpi/openmpi-4.1.6
[user@athena1 ~]$ mpicc -fopenmp -o mpi_multinode_ompthreads mpi_multinode_ompthreads.c
[user@athena1 ~]$ sbatch mpi_multinode_ompthreads_slurm_script.sh
Submitted batch job 123
[user@athena1 ~]$ squeue -u $USER
JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON) ## Job is running with JOBID 123
123 cpu MPI_multinode_ompthreads user R 0:01 2 athena[101,201]
[user@athena1 ~]$ squeue -u $USER
JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON) ## JOBID 123 completed and exited the queue
[user@athena1 ~]$ cat MPI_multinode_ompthreads_output.log
Hello World from process 0 of 4. Thread 0 of 3. Running on node athena101
Hello World from process 0 of 4. Thread 1 of 3. Running on node athena101
Hello World from process 0 of 4. Thread 2 of 3. Running on node athena101
Hello World from process 1 of 4. Thread 0 of 3. Running on node athena101
Hello World from process 1 of 4. Thread 1 of 3. Running on node athena101
Hello World from process 1 of 4. Thread 2 of 3. Running on node athena101
Hello World from process 2 of 4. Thread 0 of 3. Running on node athena201
Hello World from process 2 of 4. Thread 1 of 3. Running on node athena201
Hello World from process 2 of 4. Thread 2 of 3. Running on node athena201
Hello World from process 3 of 4. Thread 0 of 3. Running on node athena201
Hello World from process 3 of 4. Thread 1 of 3. Running on node athena201
Hello World from process 3 of 4. Thread 2 of 3. Running on node athena201