First Push

NVIDIA-Linux-x86_64-535.161.07-grid/kernel-open/nvidia/nvlink_linux.c

@@ -0,0 +1,641 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2015-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include "conftest.h"
+
+#include "nvlink_os.h"
+#include "nvlink_linux.h"
+#include "nvlink_errors.h"
+#include "nvlink_export.h"
+#include "nv-linux.h"
+#include "nv-procfs.h"
+#include "nv-time.h"
+#include "nvlink_caps.h"
+
+#include <linux/module.h>
+#include <linux/major.h>
+#include <linux/cdev.h>
+#include <linux/device.h>
+#include <linux/string.h>
+#include <linux/mutex.h>
+
+#define MAX_ERROR_STRING           512
+
+typedef struct nvlink_file_private
+{
+    struct
+    {
+        /* A duped file descriptor for fabric_mgmt capability */
+        int fabric_mgmt;
+    } capability_fds;
+} nvlink_file_private_t;
+
+#define NVLINK_SET_FILE_PRIVATE(filp, data) ((filp)->private_data = (data))
+#define NVLINK_GET_FILE_PRIVATE(filp) ((nvlink_file_private_t *)(filp)->private_data)
+
+typedef struct
+{
+    struct mutex    lock;
+    NvBool          initialized;
+    struct cdev     cdev;
+    dev_t           devno;
+    int             opened;
+    int             major_devnum;
+} _nvlink_drvctx;
+
+
+// nvlink driver local state
+static _nvlink_drvctx nvlink_drvctx;
+
+#if defined(CONFIG_PROC_FS)
+#define NV_DEFINE_SINGLE_NVLINK_PROCFS_FILE(name) \
+    NV_DEFINE_SINGLE_PROCFS_FILE_READ_ONLY(name, nv_system_pm_lock)
+#endif
+
+#define NVLINK_PROCFS_DIR "driver/nvidia-nvlink"
+
+static struct proc_dir_entry *nvlink_procfs_dir = NULL;
+
+#if defined(CONFIG_PROC_FS)
+    static int nvlink_is_procfs_available = 1;
+#else
+    static int nvlink_is_procfs_available = 0;
+#endif
+
+static struct proc_dir_entry *nvlink_permissions = NULL;
+
+static int nv_procfs_read_permissions(struct seq_file *s, void *v)
+{
+    // Restrict device node permissions - 0666.
+    seq_printf(s, "%s: %u\n", "DeviceFileMode", 438);
+
+    return 0;
+}
+
+NV_DEFINE_SINGLE_NVLINK_PROCFS_FILE(permissions);
+
+static void nvlink_permissions_exit(void)
+{
+    if (!nvlink_permissions)
+    {
+        return;
+    }
+
+    proc_remove(nvlink_permissions);
+    nvlink_permissions = NULL;
+}
+
+static int nvlink_permissions_init(void)
+{
+    if (!nvlink_procfs_dir)
+    {
+        return -EACCES;
+    }
+
+    nvlink_permissions = NV_CREATE_PROC_FILE("permissions",
+                                             nvlink_procfs_dir,
+                                             permissions,
+                                             NULL);
+    if (!nvlink_permissions)
+    {
+        return -EACCES;
+    }
+
+    return 0;
+}
+
+static void nvlink_procfs_exit(void)
+{
+    nvlink_permissions_exit();
+
+    if (!nvlink_procfs_dir)
+    {
+        return;
+    }
+
+    proc_remove(nvlink_procfs_dir);
+    nvlink_procfs_dir = NULL;
+}
+
+static int nvlink_procfs_init(void)
+{
+    int rc = 0;
+
+    if (!nvlink_is_procfs_available)
+    {
+        return -EACCES;
+    }
+
+    nvlink_procfs_dir = NV_CREATE_PROC_DIR(NVLINK_PROCFS_DIR, NULL);
+    if (!nvlink_procfs_dir)
+    {
+        return -EACCES;
+    }
+
+    rc = nvlink_permissions_init();
+    if (rc < 0)
+    {
+        goto cleanup;
+    }
+
+    return 0;
+
+cleanup:
+
+    nvlink_procfs_exit();
+
+    return rc;
+}
+
+static int nvlink_fops_open(struct inode *inode, struct file *filp)
+{
+    int rc = 0;
+    nvlink_file_private_t *private = NULL;
+
+    nvlink_print(NVLINK_DBG_INFO, "nvlink driver open\n");
+
+    mutex_lock(&nvlink_drvctx.lock);
+
+    // nvlink lib driver is currently exclusive open.
+    if (nvlink_drvctx.opened)
+    {
+        rc = -EBUSY;
+        goto open_error;
+    }
+
+    private = (nvlink_file_private_t *)nvlink_malloc(sizeof(*private));
+    if (private == NULL)
+    {
+        rc = -ENOMEM;
+        goto open_error;
+    }
+
+    private->capability_fds.fabric_mgmt = -1;
+    NVLINK_SET_FILE_PRIVATE(filp, private);
+
+    // mark our state as opened
+    nvlink_drvctx.opened = NV_TRUE;
+
+open_error:
+    mutex_unlock(&nvlink_drvctx.lock);
+    return rc;
+}
+
+static int nvlink_fops_release(struct inode *inode, struct file *filp)
+{
+    nvlink_file_private_t *private = NVLINK_GET_FILE_PRIVATE(filp);
+
+    nvlink_print(NVLINK_DBG_INFO, "nvlink driver close\n");
+
+    if (private == NULL)
+        return -ENOMEM;
+
+    mutex_lock(&nvlink_drvctx.lock);
+
+    if (private->capability_fds.fabric_mgmt > 0)
+    {
+        nvlink_cap_release(private->capability_fds.fabric_mgmt);
+        private->capability_fds.fabric_mgmt = -1;
+    }
+
+    nvlink_free(filp->private_data);
+    NVLINK_SET_FILE_PRIVATE(filp, NULL);
+
+    // mark the device as not opened
+    nvlink_drvctx.opened = NV_FALSE;
+
+    mutex_unlock(&nvlink_drvctx.lock);
+
+    return 0;
+}
+
+static int nvlink_fops_ioctl(struct inode *inode,
+                             struct file *filp,
+                             unsigned int cmd,
+                             unsigned long arg)
+{
+    nvlink_ioctrl_params ctrl_params = {0};
+    int param_size = _IOC_SIZE(cmd);
+    void *param_buf = NULL;
+    NvlStatus ret_val = 0;
+    int rc = 0;
+
+    // no buffer for simple _IO types
+    if (param_size)
+    {
+        // allocate a buffer to hold user input
+        param_buf = kzalloc(param_size, GFP_KERNEL);
+        if (NULL == param_buf) 
+        {
+            rc = -ENOMEM;
+            goto nvlink_ioctl_fail;
+        }
+
+        // copy user input to kernel buffers. Simple _IOR() ioctls can skip this step.
+        if (_IOC_DIR(cmd) & _IOC_WRITE)
+        {
+            // copy user input to local buffer
+            if (copy_from_user(param_buf, (const void *)arg, param_size))
+            {
+                rc = -EFAULT;
+                goto nvlink_ioctl_fail;
+            }
+        }
+    }
+
+    ctrl_params.osPrivate = filp->private_data;
+    ctrl_params.cmd = _IOC_NR(cmd);
+    ctrl_params.buf = param_buf;
+    ctrl_params.size = param_size;
+
+    ret_val = nvlink_lib_ioctl_ctrl(&ctrl_params);
+    if (NVL_SUCCESS != ret_val)
+    {
+        rc = -EINVAL;
+        goto nvlink_ioctl_fail;
+    }
+
+    //  no copy for write-only ioctl
+    if ((param_size) && (_IOC_DIR(cmd) & _IOC_READ))
+    {
+        if (copy_to_user((void *)arg, ctrl_params.buf, ctrl_params.size))
+        {
+            rc = -EFAULT;
+            goto nvlink_ioctl_fail;
+        }
+    }
+
+nvlink_ioctl_fail:
+    if (param_buf) 
+    {
+        kfree(param_buf);
+    }
+    return rc;
+}
+
+#define NV_FILE_INODE(file) (file)->f_inode
+
+static long nvlink_fops_unlocked_ioctl(struct file *file,
+                                       unsigned int cmd,
+                                       unsigned long arg)
+{
+    return nvlink_fops_ioctl(NV_FILE_INODE(file), file, cmd, arg);
+}
+
+
+static const struct file_operations nvlink_fops = {
+    .owner           = THIS_MODULE,
+    .open            = nvlink_fops_open,
+    .release         = nvlink_fops_release,
+    .unlocked_ioctl  = nvlink_fops_unlocked_ioctl,
+};
+
+int __init nvlink_core_init(void)
+{
+    NvlStatus ret_val;
+    int rc;
+
+    if (NV_TRUE == nvlink_drvctx.initialized)
+    {
+        nvlink_print(NVLINK_DBG_ERRORS, "nvlink core interface already initialized\n");
+        return -EBUSY;
+    }
+
+    mutex_init(&nvlink_drvctx.lock);
+
+    ret_val = nvlink_lib_initialize();
+    if (NVL_SUCCESS != ret_val)
+    {
+        nvlink_print(NVLINK_DBG_ERRORS,  "Failed to initialize driver : %d\n", ret_val);
+        rc = -ENODEV;
+        goto nvlink_lib_initialize_fail;
+    }
+
+    rc = alloc_chrdev_region(&nvlink_drvctx.devno, 0, NVLINK_NUM_MINOR_DEVICES,
+                             NVLINK_DEVICE_NAME);
+    if (rc < 0)
+    {
+        nvlink_print(NVLINK_DBG_ERRORS, "alloc_chrdev_region failed: %d\n", rc);
+        goto alloc_chrdev_region_fail;
+    }
+
+    nvlink_drvctx.major_devnum = MAJOR(nvlink_drvctx.devno);
+    nvlink_print(NVLINK_DBG_INFO, "Nvlink Core is being initialized, major device number %d\n",
+                 nvlink_drvctx.major_devnum);
+
+    cdev_init(&nvlink_drvctx.cdev, &nvlink_fops);
+    nvlink_drvctx.cdev.owner = THIS_MODULE;
+    rc = cdev_add(&nvlink_drvctx.cdev, nvlink_drvctx.devno, NVLINK_NUM_MINOR_DEVICES);
+    if (rc < 0)
+    {
+        nvlink_print(NVLINK_DBG_ERRORS, " Unable to create cdev\n");
+        goto cdev_add_fail;
+    }
+
+    rc = nvlink_procfs_init();
+    if (rc < 0)
+    {
+        goto procfs_init_fail;
+    }
+
+    rc = nvlink_cap_init(NVLINK_PROCFS_DIR);
+    if (rc < 0)
+    {
+        nvlink_print(NVLINK_DBG_ERRORS, " Unable to create capability\n");
+        goto cap_init_fail;
+    }
+
+    nvlink_drvctx.initialized = NV_TRUE;
+
+    return 0;
+
+cap_init_fail:
+    nvlink_procfs_exit();
+
+procfs_init_fail:
+    cdev_del(&nvlink_drvctx.cdev);
+
+cdev_add_fail:
+    unregister_chrdev_region(nvlink_drvctx.devno, NVLINK_NUM_MINOR_DEVICES);
+
+alloc_chrdev_region_fail:
+    nvlink_lib_unload();
+
+nvlink_lib_initialize_fail:
+    nv_mutex_destroy(&nvlink_drvctx.lock);
+    return rc;
+}
+
+void nvlink_core_exit(void)
+{
+    if (NV_FALSE == nvlink_drvctx.initialized)
+    {
+        return;
+    }
+
+    nvlink_cap_exit();
+
+    nvlink_procfs_exit();
+
+    cdev_del(&nvlink_drvctx.cdev);
+
+    unregister_chrdev_region(nvlink_drvctx.devno, NVLINK_NUM_MINOR_DEVICES);
+
+    nvlink_lib_unload();
+
+    nv_mutex_destroy(&nvlink_drvctx.lock);
+
+    nvlink_print(NVLINK_DBG_INFO, "Unregistered Nvlink Core, major device number %d\n",
+                 nvlink_drvctx.major_devnum);
+}
+
+void
+nvlink_print
+(
+    const char *file,
+    int         line,
+    const char *function,
+    int         log_level,
+    const char *fmt,
+    ...
+)
+{
+    va_list arglist;
+    char    nv_string[MAX_ERROR_STRING];
+    char   *sys_log_level;
+
+    switch (log_level) {
+    case NVLINK_DBG_LEVEL_INFO:
+        sys_log_level = KERN_INFO;
+        break;
+    case NVLINK_DBG_LEVEL_SETUP:
+        sys_log_level = KERN_DEBUG;
+        break;
+    case NVLINK_DBG_LEVEL_USERERRORS:
+        sys_log_level = KERN_NOTICE;
+        break;
+    case NVLINK_DBG_LEVEL_WARNINGS:
+        sys_log_level = KERN_WARNING;
+        break;
+    case NVLINK_DBG_LEVEL_ERRORS:
+        sys_log_level = KERN_ERR;
+        break;
+    default:
+        sys_log_level = KERN_INFO;
+        break;
+    }
+
+    va_start(arglist, fmt);
+    vsnprintf(nv_string, sizeof(nv_string), fmt, arglist);
+    va_end(arglist);
+
+    nv_string[sizeof(nv_string) - 1] = '\0';
+    printk("%snvidia-nvlink: %s", sys_log_level, nv_string);
+}
+
+void * nvlink_malloc(NvLength size)
+{
+   return kmalloc(size, GFP_KERNEL);
+}
+
+void nvlink_free(void *ptr)
+{
+    return kfree(ptr);
+}
+
+char * nvlink_strcpy(char *dest, const char *src)
+{
+    return strcpy(dest, src);
+}
+
+int nvlink_strcmp(const char *dest, const char *src)
+{
+    return strcmp(dest, src);
+}
+
+NvLength nvlink_strlen(const char *s)
+{
+    return strlen(s);
+}
+
+int nvlink_snprintf(char *dest, NvLength size, const char *fmt, ...)
+{
+    va_list arglist;
+    int chars_written;
+
+    va_start(arglist, fmt);
+    chars_written = vsnprintf(dest, size, fmt, arglist);
+    va_end(arglist);
+
+    return chars_written;
+}
+
+NvU32 nvlink_memRd32(const volatile void * address)
+{
+    return (*(const volatile NvU32*)(address));
+}
+
+void nvlink_memWr32(volatile void *address, NvU32 data)
+{
+    (*(volatile NvU32 *)(address)) = data;
+}
+
+NvU64 nvlink_memRd64(const volatile void * address)
+{
+    return (*(const volatile NvU64 *)(address));
+}
+
+void nvlink_memWr64(volatile void *address, NvU64 data)
+{
+    (*(volatile NvU64 *)(address)) = data;
+}
+
+void * nvlink_memset(void *dest, int value, NvLength size)
+{
+     return memset(dest, value, size);
+}
+
+void * nvlink_memcpy(void *dest, const void *src, NvLength size)
+{
+    return memcpy(dest, src, size);
+}
+
+int nvlink_memcmp(const void *s1, const void *s2, NvLength size)
+{
+    return memcmp(s1, s2, size);
+}
+
+/*
+ * Sleep for specified milliseconds. Yields the CPU to scheduler.
+ */
+void nvlink_sleep(unsigned int ms)
+{
+    NV_STATUS status;
+
+    status = nv_sleep_ms(ms);
+
+    if (status !=  NV_OK)
+    {
+        if (printk_ratelimit())
+        {
+            nvlink_print(NVLINK_DBG_ERRORS, "NVLink: requested sleep duration"
+                         " %d msec exceeded %d msec\n",
+                         ms, NV_MAX_ISR_DELAY_MS);
+            WARN_ON(1);
+        }
+    }
+}
+
+void nvlink_assert(int cond)
+{
+    if ((cond) == 0x0)
+    {
+        if (printk_ratelimit())
+        {
+            nvlink_print(NVLINK_DBG_ERRORS, "NVLink: Assertion failed!\n");
+            WARN_ON(1);
+        }
+
+        dbg_breakpoint();
+    }
+}
+
+void * nvlink_allocLock(void)
+{
+    struct semaphore *sema;
+
+    sema = nvlink_malloc(sizeof(*sema));
+    if (sema == NULL)
+    {
+        nvlink_print(NVLINK_DBG_ERRORS, "Failed to allocate sema!\n");
+        return NULL;
+    }
+    sema_init(sema, 1);
+
+    return sema;
+}
+
+void nvlink_acquireLock(void *hLock)
+{
+    down(hLock);
+}
+
+void nvlink_releaseLock(void *hLock)
+{
+    up(hLock);
+}
+
+void nvlink_freeLock(void *hLock)
+{
+    if (NULL == hLock)
+    {
+        return;
+    }
+
+    NVLINK_FREE(hLock);
+}
+
+NvBool nvlink_isLockOwner(void *hLock)
+{
+    return NV_TRUE;
+}
+
+NvlStatus nvlink_acquire_fabric_mgmt_cap(void *osPrivate, NvU64 capDescriptor)
+{
+    int dup_fd = -1;
+    nvlink_file_private_t *private_data = (nvlink_file_private_t *)osPrivate;
+
+    if (private_data == NULL)
+    {
+        return NVL_BAD_ARGS;
+    }
+
+    dup_fd = nvlink_cap_acquire((int)capDescriptor,
+                                NVLINK_CAP_FABRIC_MANAGEMENT);
+    if (dup_fd < 0)
+    {
+        return NVL_ERR_OPERATING_SYSTEM;
+    }
+
+    private_data->capability_fds.fabric_mgmt = dup_fd;
+    return NVL_SUCCESS;
+}
+
+int nvlink_is_fabric_manager(void *osPrivate)
+{
+    nvlink_file_private_t *private_data = (nvlink_file_private_t *)osPrivate;
+
+    /* Make sure that fabric mgmt capbaility fd is valid */
+    if ((private_data == NULL) ||
+        (private_data->capability_fds.fabric_mgmt < 0))
+    {
+        return 0;
+    }
+
+    return 1;
+}
+
+int nvlink_is_admin(void)
+{
+    return NV_IS_SUSER();
+}