CUDA Libraries

cuBLAS / cuBLASLt / cuBLASXt dense linear algebra (BLAS), Tensor Core GEMMs, multi-GPU sparse matrix operations (SpMV, SpMM, CSR/COO/BSR)

LU, QR, Cholesky, eigen/SVD solvers (dense & sparse) fast Fourier transforms (1-D/2-D/3-D, batched) random number generation (host & device APIs) tensor contractions, reductions, permutations deep neural network primitives (convolutions, attention, normalization) optimized deep learning inference multi-GPU / multi-node collectives (all-reduce, broadcast) image & signal processing primitives nvJPEG / NVDEC / NVENC image/video decode & encode CUTLASS / CuTe

C++ templates for custom high-performance GEMMs/convolutions cuDSS, AmgX sparse direct solvers, algebraic multigrid

cuQuantum, cuDF/cuML/cuGraph (RAPIDS), cuVS, Warp, Modulus

30.1 cuBLAS SGEMM

#include <cublas_v2.h>
cublasHandle_t handle;
cublasCreate(&handle);
const float alpha = 1.0f, beta = 0.0f;
// cuBLAS uses COLUMN-MAJOR storage. For row-major C = A×B, compute Cᵀ = Bᵀ×Aᵀ:
cublasSgemm(handle, CUBLAS_OP_N, CUBLAS_OP_N,
            N, M, K, &alpha,
            d_B, N,      // B (row-major K×N) seen as column-major N×K
            d_A, K,      // A (row-major M×K) seen as column-major K×M
            &beta, d_C, N);
cublasSetStream(handle, stream);
cublasSetMathMode(handle, CUBLAS_TF32_TENSOR_OP_MATH);   // allow TF32 Tensor Cores
cublasGemmEx(...);                                       // mixed precision (FP16/BF16 in, FP32 out)
cublasDestroy(handle);
// link: -lcublas

30.2 cuFFT

#include <cufft.h>
cufftHandle plan;
cufftPlan1d(&plan, N, CUFFT_C2C, batch);
cufftExecC2C(plan, d_signal, d_signal, CUFFT_FORWARD);   // in-place forward FFT
cufftExecC2C(plan, d_signal, d_signal, CUFFT_INVERSE);   // unnormalized inverse
cufftDestroy(plan);
// link: -lcufft

30.3 cuRAND

#include <curand.h>
curandGenerator_t gen;
curandCreateGenerator(&gen, CURAND_RNG_PSEUDO_PHILOX4_32_10);
curandSetPseudoRandomGeneratorSeed(gen, 1234ULL);
curandGenerateUniform(gen, d_rand, n);       // host API: fill device array
curandGenerateNormal(gen, d_norm, n, 0.0f, 1.0f);

#include <curand_kernel.h>                   // device API: per-thread generators
__global__ void monteCarloPi(unsigned long long seed, unsigned long long* hits, int perThread) {
    int id = blockIdx.x * blockDim.x + threadIdx.x;
    curandStatePhilox4_32_10_t state;
    curand_init(seed, id, 0, &state);
    unsigned long long local = 0;
    for (int i = 0; i < perThread; ++i) {
        float x = curand_uniform(&state), y = curand_uniform(&state);
        local += (x * x + y * y <= 1.0f);
    }
    atomicAdd(hits, local);
}

30.4 cuSOLVER (LU solve)

cusolverDnHandle_t h; cusolverDnCreate(&h);
int lwork; cusolverDnSgetrf_bufferSize(h, n, n, d_A, n, &lwork);
cusolverDnSgetrf(h, n, n, d_A, n, d_work, d_ipiv, d_info);             // LU factorization
cusolverDnSgetrs(h, CUBLAS_OP_N, n, 1, d_A, n, d_ipiv, d_b, n, d_info); // solve