VASP
运行环境
CentOS-7-x86_64-DVD，Intel® Parallel Studio XE 2020。
CentOS安装
修改makefile，makefile.include，src/constr_cell_relax.F，其中makefile.include，
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
|
# Precompiler options
CPP_OPTIONS= -DHOST=\"LinuxIFC\"\
-DMPI -DMPI_BLOCK=8000 \
-Duse_collective \
-DscaLAPACK \
-DCACHE_SIZE=4000 \
-Davoidalloc \
-Duse_bse_te \
-Dtbdyn \
-Duse_shmem
CPP = fpp -f_com=no -free -w0 $*$(FUFFIX) $*$(SUFFIX) $(CPP_OPTIONS)
FC = mpiifort
FCL = mpiifort -mkl=cluster -lstdc++
FREE = -free -names lowercase
FFLAGS = -assume byterecl -w -xHOST -heap-arrays 64
OFLAG = -O2 -xHOST
OFLAG_IN = $(OFLAG)
DEBUG = -O0
MKLROOT = /opt/ohpc/pub/apps/intel2022/compilers_and_libraries_2020.4.304/linux/mkl
MKL_PATH = $(MKLROOT)/lib/intel64
BLAS =
LAPACK =
BLACS = -lmkl_blacs_intelmpi_lp64
SCALAPACK = $(MKL_PATH)/libmkl_scalapack_lp64.a $(BLACS)
OBJECTS = fftmpiw.o fftmpi_map.o fft3dlib.o fftw3d.o
INCS =-I$(MKLROOT)/include/fftw
LLIBS = $(SCALAPACK) $(LAPACK) $(BLAS)
OBJECTS_O1 += fftw3d.o fftmpi.o fftmpiw.o
OBJECTS_O2 += fft3dlib.o
# For what used to be vasp.5.lib
CPP_LIB = $(CPP)
FC_LIB = $(FC)
CC_LIB = icc
CFLAGS_LIB = -O
FFLAGS_LIB = -O1
FREE_LIB = $(FREE)
OBJECTS_LIB= linpack_double.o getshmem.o
# For the parser library
CXX_PARS = icpc
LIBS += parser
LLIBS += -Lparser -lparser -lstdc++
# Normally no need to change this
SRCDIR = ../../src
BINDIR = ../../bin
#================================================
# GPU Stuff
CPP_GPU = -DCUDA_GPU -DRPROMU_CPROJ_OVERLAP -DUSE_PINNED_MEMORY -DCUFFT_MIN=28 -UscaLAPACK
OBJECTS_GPU = fftmpiw.o fftmpi_map.o fft3dlib.o fftw3d_gpu.o fftmpiw_gpu.o
CC = icc
CXX = icpc
CFLAGS = -fPIC -DADD_ -Wall -openmp -DMAGMA_WITH_MKL -DMAGMA_SETAFFINITY -DGPUSHMEM=300 -DHAVE_CUBLAS
CUDA_ROOT ?= /usr/local/cuda/
NVCC := $(CUDA_ROOT)/bin/nvcc -ccbin=icc
CUDA_LIB := -L$(CUDA_ROOT)/lib64 -lnvToolsExt -lcudart -lcuda -lcufft -lcublas
GENCODE_ARCH := -gencode=arch=compute_30,code=\"sm_30,compute_30\" \
-gencode=arch=compute_35,code=\"sm_35,compute_35\" \
-gencode=arch=compute_60,code=\"sm_60,compute_60\"
MPI_INC = $(I_MPI_ROOT)/include64/
|
安装，
1
2
|
source /opt/ohpc/pub/apps/intel2022/parallel_studio_xe_2020.4.912/psxevars.sh intel64 >/dev/null 2>&1
chmod -R 755 ./* && make all #不要使用make all -j *
|
修改.bashrc，
1
2
|
#vasp
export PATH=$PATH:/opt/ohpc/pub/apps/vasp5.4.4/bin
|
运行
1
2
3
4
|
ulimit -s unlimited #与FFLAGS = -heap-arrays 64等价
export I_MPI_ADJUST_REDUCE=3 # https://www.nsc.liu.se/software/installed/tetralith/vasp/
mpirun -np 2 vasp_std > log
mpirun -np 2 vasp_std > log & #后台运行
|
Ubuntu安装
参考此处，或安装依赖，
1
2
3
4
5
6
|
#vasp
sudo apt install -y gfortran gcc g++
sudo apt install -y libblas-dev liblapack-dev
sudo apt install -y libfftw3-dev libfftw3-mpi-dev
sudo apt install -y openmpi-bin libopenmpi-dev
sudo apt install -y libscalapack-mpi-dev
|
修改makefile.include，
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
|
CPP_OPTIONS= -DHOST=\"LinuxGNU\" \
-DMPI -DMPI_BLOCK=8000 \
-Duse_collective \
-DscaLAPACK \
-DCACHE_SIZE=4000 \
-Davoidalloc \
-Duse_bse_te \
-Dtbdyn \
-Duse_shmem
CPP = gcc -E -P -C -w $*$(FUFFIX) >$*$(SUFFIX) $(CPP_OPTIONS)
FC = mpif90
FCL = mpif90 -no-pie
FREE = -ffree-form -ffree-line-length-none
FFLAGS = -w -fallow-argument-mismatch
OFLAG = -O2
OFLAG_IN = $(OFLAG)
DEBUG = -O0
LIBDIR =
BLAS = -lblas
LAPACK = -llapack
BLACS =
SCALAPACK = -lscalapack-openmpi $(BLACS)
LLIBS = $(SCALAPACK) $(LAPACK) $(BLAS)
FFTW ?= /usr
LLIBS += -L$(FFTW)/lib -lfftw3_mpi -lfftw3
INCS = -I$(FFTW)/include
OBJECTS = fftmpiw.o fftmpi_map.o fftw3d.o fft3dlib.o
OBJECTS_O1 += fftw3d.o fftmpi.o fftmpiw.o
OBJECTS_O2 += fft3dlib.o
CPP_LIB = $(CPP)
FC_LIB = $(FC)
CC_LIB = gcc
CFLAGS_LIB = -O
FFLAGS_LIB = -O1 -fallow-argument-mismatch
FREE_LIB = $(FREE)
OBJECTS_LIB= linpack_double.o getshmem.o
CXX_PARS = g++
LIBS += parser
LLIBS += -Lparser -lparser -lstdc++
SRCDIR = ../../src
BINDIR = ../../bin
CC = gcc
CXX = g++
CFLAGS = -O2 -fPIC -fopenmp
|
job.sh
1
2
3
4
5
6
7
8
|
#!/bin/bash
#SBATCH --job-name=vasp_std
#SBATCH --output=%j.out
#SBATCH --partition=debug
#SBATCH --ntasks=8
#SBATCH --exclusive
mpirun vasp_std > log
|
与CentOS性能对比
-
Ubuntu (Ubuntu 24.04.3 LTS, Intel(R) Core(TM) Ultra 9 285K CPU @ 5.3GHz, slurm调用了8个大核，128Gb内存，GCC): 948s
-
CentOS node01-04 (CentOS Linux 7 (Core), 英特尔® 至强® Gold 6240 处理器 x 2，内核数 18 x 2，处理器基本频率 2.60 GHz，最大睿频频率 3.90 GHz，缓存 24.75 MB, 192Gb内存，Intel Parallel Studio XE Cluster Edition 2020 Update 4): 145s
-
CentOS node05-06 (CentOS Linux 7 (Core), 英特尔® 至强® Gold 6242R 处理器 x 2，内核数 20 x 2，处理器基本频率 3.10 GHz，最大睿频频率 4.10 GHz，缓存 35.75 MB, 192Gb内存，Intel Parallel Studio XE Cluster Edition 2020 Update 4): 120s
-
CentOS node07 (CentOS Linux 7 (Core), 英特尔® 至强® 金牌 6444Y 处理器 x 2, 内核数 16 x 2，处理器基本频率 3.60 GHz，最大睿频频率 4.00 GHz，缓存 45 MB，256Gb内存，Intel Parallel Studio XE Cluster Edition 2020 Update 4): 77s
注意
Ubuntu慢的原因：
核数少、内存小，使用GCC而非Intel编译器。
其实主频还是挺高的。若是使用Intel编译器并使用全部核数，估计可能在200s左右。
VASPsol
1
2
3
4
5
6
7
8
9
10
11
|
# Precompiler options
CPP_OPTIONS= -DHOST=\"LinuxIFC\"\
-DMPI -DMPI_BLOCK=8000 \
-Duse_collective \
-DscaLAPACK \
-DCACHE_SIZE=4000 \
-Davoidalloc \
-Duse_bse_te \
-Dtbdyn \
-Duse_shmem \
-Dsol_compat
|
VTST
1
2
|
CALL CHAIN_FORCE(T_INFO%NIONS,DYN%POSION,TOTEN,TIFOR, &
LATT_CUR%A,LATT_CUR%B,IO%IU6)
|
替换为
1
2
|
CALL CHAIN_FORCE(T_INFO%NIONS,DYN%POSION,TOTEN,TIFOR, &
TSIF,LATT_CUR%A,LATT_CUR%B,IO%IU6)
|
- 在
src/.objects中的chain.o前面增加，
1
2
|
bfgs.o dynmat.o instanton.o lbfgs.o sd.o cg.o dimer.o bbm.o \
fire.o lanczos.o neb.o qm.o opt.o \
|
Wannier90
1
|
LIBDIR = /opt/ohpc/pub/apps/intel2022/compilers_and_libraries_2020.4.304/linux/mkl/lib/intel64
|
make all。
- 若vasp5.4.4与wannier90联用，需要修改
makefile.include，
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
|
# Precompiler options
CPP_OPTIONS= -DHOST=\"LinuxIFC\"\
-DMPI -DMPI_BLOCK=8000 \
-Duse_collective \
-DscaLAPACK \
-DCACHE_SIZE=4000 \
-Davoidalloc \
-Duse_bse_te \
-Dtbdyn \
-Duse_shmem \
-DVASP2WANNIER90
CPP = fpp -f_com=no -free -w0 $*$(FUFFIX) $*$(SUFFIX) $(CPP_OPTIONS)
FC = mpiifort
FCL = mpiifort -mkl=cluster -lstdc++
FREE = -free -names lowercase
FFLAGS = -assume byterecl -w -xHOST -heap-arrays 64
OFLAG = -O2 -xHOST
OFLAG_IN = $(OFLAG)
DEBUG = -O0
MKLROOT = /opt/ohpc/pub/apps/intel2022/compilers_and_libraries_2020.4.304/linux/mkl
MKL_PATH = $(MKLROOT)/lib/intel64
BLAS =
LAPACK =
BLACS = -lmkl_blacs_intelmpi_lp64
SCALAPACK = $(MKL_PATH)/libmkl_scalapack_lp64.a $(BLACS)
OBJECTS = fftmpiw.o fftmpi_map.o fft3dlib.o fftw3d.o
INCS =-I$(MKLROOT)/include/fftw
LLIBS = $(SCALAPACK) $(LAPACK) $(BLAS)
LLIBS += /opt/ohpc/pub/apps/vasp5.4.4_wannier90/wannier90-3.1.0/libwannier.a
OBJECTS_O1 += fftw3d.o fftmpi.o fftmpiw.o
OBJECTS_O2 += fft3dlib.o
# For what used to be vasp.5.lib
CPP_LIB = $(CPP)
FC_LIB = $(FC)
CC_LIB = icc
CFLAGS_LIB = -O
FFLAGS_LIB = -O1
FREE_LIB = $(FREE)
OBJECTS_LIB= linpack_double.o getshmem.o
# For the parser library
CXX_PARS = icpc
LIBS += parser
LLIBS += -Lparser -lparser -lstdc++
# Normally no need to change this
SRCDIR = ../../src
BINDIR = ../../bin
#================================================
# GPU Stuff
CPP_GPU = -DCUDA_GPU -DRPROMU_CPROJ_OVERLAP -DUSE_PINNED_MEMORY -DCUFFT_MIN=28 -UscaLAPACK
OBJECTS_GPU = fftmpiw.o fftmpi_map.o fft3dlib.o fftw3d_gpu.o fftmpiw_gpu.o
CC = icc
CXX = icpc
CFLAGS = -fPIC -DADD_ -Wall -openmp -DMAGMA_WITH_MKL -DMAGMA_SETAFFINITY -DGPUSHMEM=300 -DHAVE_CUBLAS
CUDA_ROOT ?= /usr/local/cuda/
NVCC := $(CUDA_ROOT)/bin/nvcc -ccbin=icc
CUDA_LIB := -L$(CUDA_ROOT)/lib64 -lnvToolsExt -lcudart -lcuda -lcufft -lcublas
GENCODE_ARCH := -gencode=arch=compute_30,code=\"sm_30,compute_30\" \
-gencode=arch=compute_35,code=\"sm_35,compute_35\" \
-gencode=arch=compute_60,code=\"sm_60,compute_60\"
MPI_INC = $(I_MPI_ROOT)/include64/
|
文档
