sx8.c 40 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749
  1. /*
  2. * sx8.c: Driver for Promise SATA SX8 looks-like-I2O hardware
  3. *
  4. * Copyright 2004-2005 Red Hat, Inc.
  5. *
  6. * Author/maintainer: Jeff Garzik <jgarzik@pobox.com>
  7. *
  8. * This file is subject to the terms and conditions of the GNU General Public
  9. * License. See the file "COPYING" in the main directory of this archive
  10. * for more details.
  11. */
  12. #include <linux/kernel.h>
  13. #include <linux/module.h>
  14. #include <linux/init.h>
  15. #include <linux/pci.h>
  16. #include <linux/slab.h>
  17. #include <linux/spinlock.h>
  18. #include <linux/blkdev.h>
  19. #include <linux/sched.h>
  20. #include <linux/interrupt.h>
  21. #include <linux/compiler.h>
  22. #include <linux/workqueue.h>
  23. #include <linux/bitops.h>
  24. #include <linux/delay.h>
  25. #include <linux/ktime.h>
  26. #include <linux/hdreg.h>
  27. #include <linux/dma-mapping.h>
  28. #include <linux/completion.h>
  29. #include <linux/scatterlist.h>
  30. #include <asm/io.h>
  31. #include <asm/uaccess.h>
  32. #if 0
  33. #define CARM_DEBUG
  34. #define CARM_VERBOSE_DEBUG
  35. #else
  36. #undef CARM_DEBUG
  37. #undef CARM_VERBOSE_DEBUG
  38. #endif
  39. #undef CARM_NDEBUG
  40. #define DRV_NAME "sx8"
  41. #define DRV_VERSION "1.0"
  42. #define PFX DRV_NAME ": "
  43. MODULE_AUTHOR("Jeff Garzik");
  44. MODULE_LICENSE("GPL");
  45. MODULE_DESCRIPTION("Promise SATA SX8 block driver");
  46. MODULE_VERSION(DRV_VERSION);
  47. /*
  48. * SX8 hardware has a single message queue for all ATA ports.
  49. * When this driver was written, the hardware (firmware?) would
  50. * corrupt data eventually, if more than one request was outstanding.
  51. * As one can imagine, having 8 ports bottlenecking on a single
  52. * command hurts performance.
  53. *
  54. * Based on user reports, later versions of the hardware (firmware?)
  55. * seem to be able to survive with more than one command queued.
  56. *
  57. * Therefore, we default to the safe option -- 1 command -- but
  58. * allow the user to increase this.
  59. *
  60. * SX8 should be able to support up to ~60 queued commands (CARM_MAX_REQ),
  61. * but problems seem to occur when you exceed ~30, even on newer hardware.
  62. */
  63. static int max_queue = 1;
  64. module_param(max_queue, int, 0444);
  65. MODULE_PARM_DESC(max_queue, "Maximum number of queued commands. (min==1, max==30, safe==1)");
  66. #define NEXT_RESP(idx) ((idx + 1) % RMSG_Q_LEN)
  67. /* 0xf is just arbitrary, non-zero noise; this is sorta like poisoning */
  68. #define TAG_ENCODE(tag) (((tag) << 16) | 0xf)
  69. #define TAG_DECODE(tag) (((tag) >> 16) & 0x1f)
  70. #define TAG_VALID(tag) ((((tag) & 0xf) == 0xf) && (TAG_DECODE(tag) < 32))
  71. /* note: prints function name for you */
  72. #ifdef CARM_DEBUG
  73. #define DPRINTK(fmt, args...) printk(KERN_ERR "%s: " fmt, __func__, ## args)
  74. #ifdef CARM_VERBOSE_DEBUG
  75. #define VPRINTK(fmt, args...) printk(KERN_ERR "%s: " fmt, __func__, ## args)
  76. #else
  77. #define VPRINTK(fmt, args...)
  78. #endif /* CARM_VERBOSE_DEBUG */
  79. #else
  80. #define DPRINTK(fmt, args...)
  81. #define VPRINTK(fmt, args...)
  82. #endif /* CARM_DEBUG */
  83. #ifdef CARM_NDEBUG
  84. #define assert(expr)
  85. #else
  86. #define assert(expr) \
  87. if(unlikely(!(expr))) { \
  88. printk(KERN_ERR "Assertion failed! %s,%s,%s,line=%d\n", \
  89. #expr, __FILE__, __func__, __LINE__); \
  90. }
  91. #endif
  92. /* defines only for the constants which don't work well as enums */
  93. struct carm_host;
  94. enum {
  95. /* adapter-wide limits */
  96. CARM_MAX_PORTS = 8,
  97. CARM_SHM_SIZE = (4096 << 7),
  98. CARM_MINORS_PER_MAJOR = 256 / CARM_MAX_PORTS,
  99. CARM_MAX_WAIT_Q = CARM_MAX_PORTS + 1,
  100. /* command message queue limits */
  101. CARM_MAX_REQ = 64, /* max command msgs per host */
  102. CARM_MSG_LOW_WATER = (CARM_MAX_REQ / 4), /* refill mark */
  103. /* S/G limits, host-wide and per-request */
  104. CARM_MAX_REQ_SG = 32, /* max s/g entries per request */
  105. CARM_MAX_HOST_SG = 600, /* max s/g entries per host */
  106. CARM_SG_LOW_WATER = (CARM_MAX_HOST_SG / 4), /* re-fill mark */
  107. /* hardware registers */
  108. CARM_IHQP = 0x1c,
  109. CARM_INT_STAT = 0x10, /* interrupt status */
  110. CARM_INT_MASK = 0x14, /* interrupt mask */
  111. CARM_HMUC = 0x18, /* host message unit control */
  112. RBUF_ADDR_LO = 0x20, /* response msg DMA buf low 32 bits */
  113. RBUF_ADDR_HI = 0x24, /* response msg DMA buf high 32 bits */
  114. RBUF_BYTE_SZ = 0x28,
  115. CARM_RESP_IDX = 0x2c,
  116. CARM_CMS0 = 0x30, /* command message size reg 0 */
  117. CARM_LMUC = 0x48,
  118. CARM_HMPHA = 0x6c,
  119. CARM_INITC = 0xb5,
  120. /* bits in CARM_INT_{STAT,MASK} */
  121. INT_RESERVED = 0xfffffff0,
  122. INT_WATCHDOG = (1 << 3), /* watchdog timer */
  123. INT_Q_OVERFLOW = (1 << 2), /* cmd msg q overflow */
  124. INT_Q_AVAILABLE = (1 << 1), /* cmd msg q has free space */
  125. INT_RESPONSE = (1 << 0), /* response msg available */
  126. INT_ACK_MASK = INT_WATCHDOG | INT_Q_OVERFLOW,
  127. INT_DEF_MASK = INT_RESERVED | INT_Q_OVERFLOW |
  128. INT_RESPONSE,
  129. /* command messages, and related register bits */
  130. CARM_HAVE_RESP = 0x01,
  131. CARM_MSG_READ = 1,
  132. CARM_MSG_WRITE = 2,
  133. CARM_MSG_VERIFY = 3,
  134. CARM_MSG_GET_CAPACITY = 4,
  135. CARM_MSG_FLUSH = 5,
  136. CARM_MSG_IOCTL = 6,
  137. CARM_MSG_ARRAY = 8,
  138. CARM_MSG_MISC = 9,
  139. CARM_CME = (1 << 2),
  140. CARM_RME = (1 << 1),
  141. CARM_WZBC = (1 << 0),
  142. CARM_RMI = (1 << 0),
  143. CARM_Q_FULL = (1 << 3),
  144. CARM_MSG_SIZE = 288,
  145. CARM_Q_LEN = 48,
  146. /* CARM_MSG_IOCTL messages */
  147. CARM_IOC_SCAN_CHAN = 5, /* scan channels for devices */
  148. CARM_IOC_GET_TCQ = 13, /* get tcq/ncq depth */
  149. CARM_IOC_SET_TCQ = 14, /* set tcq/ncq depth */
  150. IOC_SCAN_CHAN_NODEV = 0x1f,
  151. IOC_SCAN_CHAN_OFFSET = 0x40,
  152. /* CARM_MSG_ARRAY messages */
  153. CARM_ARRAY_INFO = 0,
  154. ARRAY_NO_EXIST = (1 << 31),
  155. /* response messages */
  156. RMSG_SZ = 8, /* sizeof(struct carm_response) */
  157. RMSG_Q_LEN = 48, /* resp. msg list length */
  158. RMSG_OK = 1, /* bit indicating msg was successful */
  159. /* length of entire resp. msg buffer */
  160. RBUF_LEN = RMSG_SZ * RMSG_Q_LEN,
  161. PDC_SHM_SIZE = (4096 << 7), /* length of entire h/w buffer */
  162. /* CARM_MSG_MISC messages */
  163. MISC_GET_FW_VER = 2,
  164. MISC_ALLOC_MEM = 3,
  165. MISC_SET_TIME = 5,
  166. /* MISC_GET_FW_VER feature bits */
  167. FW_VER_4PORT = (1 << 2), /* 1=4 ports, 0=8 ports */
  168. FW_VER_NON_RAID = (1 << 1), /* 1=non-RAID firmware, 0=RAID */
  169. FW_VER_ZCR = (1 << 0), /* zero channel RAID (whatever that is) */
  170. /* carm_host flags */
  171. FL_NON_RAID = FW_VER_NON_RAID,
  172. FL_4PORT = FW_VER_4PORT,
  173. FL_FW_VER_MASK = (FW_VER_NON_RAID | FW_VER_4PORT),
  174. FL_DAC = (1 << 16),
  175. FL_DYN_MAJOR = (1 << 17),
  176. };
  177. enum {
  178. CARM_SG_BOUNDARY = 0xffffUL, /* s/g segment boundary */
  179. };
  180. enum scatter_gather_types {
  181. SGT_32BIT = 0,
  182. SGT_64BIT = 1,
  183. };
  184. enum host_states {
  185. HST_INVALID, /* invalid state; never used */
  186. HST_ALLOC_BUF, /* setting up master SHM area */
  187. HST_ERROR, /* we never leave here */
  188. HST_PORT_SCAN, /* start dev scan */
  189. HST_DEV_SCAN_START, /* start per-device probe */
  190. HST_DEV_SCAN, /* continue per-device probe */
  191. HST_DEV_ACTIVATE, /* activate devices we found */
  192. HST_PROBE_FINISHED, /* probe is complete */
  193. HST_PROBE_START, /* initiate probe */
  194. HST_SYNC_TIME, /* tell firmware what time it is */
  195. HST_GET_FW_VER, /* get firmware version, adapter port cnt */
  196. };
  197. #ifdef CARM_DEBUG
  198. static const char *state_name[] = {
  199. "HST_INVALID",
  200. "HST_ALLOC_BUF",
  201. "HST_ERROR",
  202. "HST_PORT_SCAN",
  203. "HST_DEV_SCAN_START",
  204. "HST_DEV_SCAN",
  205. "HST_DEV_ACTIVATE",
  206. "HST_PROBE_FINISHED",
  207. "HST_PROBE_START",
  208. "HST_SYNC_TIME",
  209. "HST_GET_FW_VER",
  210. };
  211. #endif
  212. struct carm_port {
  213. unsigned int port_no;
  214. struct gendisk *disk;
  215. struct carm_host *host;
  216. /* attached device characteristics */
  217. u64 capacity;
  218. char name[41];
  219. u16 dev_geom_head;
  220. u16 dev_geom_sect;
  221. u16 dev_geom_cyl;
  222. };
  223. struct carm_request {
  224. unsigned int tag;
  225. int n_elem;
  226. unsigned int msg_type;
  227. unsigned int msg_subtype;
  228. unsigned int msg_bucket;
  229. struct request *rq;
  230. struct carm_port *port;
  231. struct scatterlist sg[CARM_MAX_REQ_SG];
  232. };
  233. struct carm_host {
  234. unsigned long flags;
  235. void __iomem *mmio;
  236. void *shm;
  237. dma_addr_t shm_dma;
  238. int major;
  239. int id;
  240. char name[32];
  241. spinlock_t lock;
  242. struct pci_dev *pdev;
  243. unsigned int state;
  244. u32 fw_ver;
  245. struct request_queue *oob_q;
  246. unsigned int n_oob;
  247. unsigned int hw_sg_used;
  248. unsigned int resp_idx;
  249. unsigned int wait_q_prod;
  250. unsigned int wait_q_cons;
  251. struct request_queue *wait_q[CARM_MAX_WAIT_Q];
  252. unsigned int n_msgs;
  253. u64 msg_alloc;
  254. struct carm_request req[CARM_MAX_REQ];
  255. void *msg_base;
  256. dma_addr_t msg_dma;
  257. int cur_scan_dev;
  258. unsigned long dev_active;
  259. unsigned long dev_present;
  260. struct carm_port port[CARM_MAX_PORTS];
  261. struct work_struct fsm_task;
  262. struct completion probe_comp;
  263. };
  264. struct carm_response {
  265. __le32 ret_handle;
  266. __le32 status;
  267. } __attribute__((packed));
  268. struct carm_msg_sg {
  269. __le32 start;
  270. __le32 len;
  271. } __attribute__((packed));
  272. struct carm_msg_rw {
  273. u8 type;
  274. u8 id;
  275. u8 sg_count;
  276. u8 sg_type;
  277. __le32 handle;
  278. __le32 lba;
  279. __le16 lba_count;
  280. __le16 lba_high;
  281. struct carm_msg_sg sg[32];
  282. } __attribute__((packed));
  283. struct carm_msg_allocbuf {
  284. u8 type;
  285. u8 subtype;
  286. u8 n_sg;
  287. u8 sg_type;
  288. __le32 handle;
  289. __le32 addr;
  290. __le32 len;
  291. __le32 evt_pool;
  292. __le32 n_evt;
  293. __le32 rbuf_pool;
  294. __le32 n_rbuf;
  295. __le32 msg_pool;
  296. __le32 n_msg;
  297. struct carm_msg_sg sg[8];
  298. } __attribute__((packed));
  299. struct carm_msg_ioctl {
  300. u8 type;
  301. u8 subtype;
  302. u8 array_id;
  303. u8 reserved1;
  304. __le32 handle;
  305. __le32 data_addr;
  306. u32 reserved2;
  307. } __attribute__((packed));
  308. struct carm_msg_sync_time {
  309. u8 type;
  310. u8 subtype;
  311. u16 reserved1;
  312. __le32 handle;
  313. u32 reserved2;
  314. __le32 timestamp;
  315. } __attribute__((packed));
  316. struct carm_msg_get_fw_ver {
  317. u8 type;
  318. u8 subtype;
  319. u16 reserved1;
  320. __le32 handle;
  321. __le32 data_addr;
  322. u32 reserved2;
  323. } __attribute__((packed));
  324. struct carm_fw_ver {
  325. __le32 version;
  326. u8 features;
  327. u8 reserved1;
  328. u16 reserved2;
  329. } __attribute__((packed));
  330. struct carm_array_info {
  331. __le32 size;
  332. __le16 size_hi;
  333. __le16 stripe_size;
  334. __le32 mode;
  335. __le16 stripe_blk_sz;
  336. __le16 reserved1;
  337. __le16 cyl;
  338. __le16 head;
  339. __le16 sect;
  340. u8 array_id;
  341. u8 reserved2;
  342. char name[40];
  343. __le32 array_status;
  344. /* device list continues beyond this point? */
  345. } __attribute__((packed));
  346. static int carm_init_one (struct pci_dev *pdev, const struct pci_device_id *ent);
  347. static void carm_remove_one (struct pci_dev *pdev);
  348. static int carm_bdev_getgeo(struct block_device *bdev, struct hd_geometry *geo);
  349. static const struct pci_device_id carm_pci_tbl[] = {
  350. { PCI_VENDOR_ID_PROMISE, 0x8000, PCI_ANY_ID, PCI_ANY_ID, 0, 0, },
  351. { PCI_VENDOR_ID_PROMISE, 0x8002, PCI_ANY_ID, PCI_ANY_ID, 0, 0, },
  352. { } /* terminate list */
  353. };
  354. MODULE_DEVICE_TABLE(pci, carm_pci_tbl);
  355. static struct pci_driver carm_driver = {
  356. .name = DRV_NAME,
  357. .id_table = carm_pci_tbl,
  358. .probe = carm_init_one,
  359. .remove = carm_remove_one,
  360. };
  361. static const struct block_device_operations carm_bd_ops = {
  362. .owner = THIS_MODULE,
  363. .getgeo = carm_bdev_getgeo,
  364. };
  365. static unsigned int carm_host_id;
  366. static unsigned long carm_major_alloc;
  367. static int carm_bdev_getgeo(struct block_device *bdev, struct hd_geometry *geo)
  368. {
  369. struct carm_port *port = bdev->bd_disk->private_data;
  370. geo->heads = (u8) port->dev_geom_head;
  371. geo->sectors = (u8) port->dev_geom_sect;
  372. geo->cylinders = port->dev_geom_cyl;
  373. return 0;
  374. }
  375. static const u32 msg_sizes[] = { 32, 64, 128, CARM_MSG_SIZE };
  376. static inline int carm_lookup_bucket(u32 msg_size)
  377. {
  378. int i;
  379. for (i = 0; i < ARRAY_SIZE(msg_sizes); i++)
  380. if (msg_size <= msg_sizes[i])
  381. return i;
  382. return -ENOENT;
  383. }
  384. static void carm_init_buckets(void __iomem *mmio)
  385. {
  386. unsigned int i;
  387. for (i = 0; i < ARRAY_SIZE(msg_sizes); i++)
  388. writel(msg_sizes[i], mmio + CARM_CMS0 + (4 * i));
  389. }
  390. static inline void *carm_ref_msg(struct carm_host *host,
  391. unsigned int msg_idx)
  392. {
  393. return host->msg_base + (msg_idx * CARM_MSG_SIZE);
  394. }
  395. static inline dma_addr_t carm_ref_msg_dma(struct carm_host *host,
  396. unsigned int msg_idx)
  397. {
  398. return host->msg_dma + (msg_idx * CARM_MSG_SIZE);
  399. }
  400. static int carm_send_msg(struct carm_host *host,
  401. struct carm_request *crq)
  402. {
  403. void __iomem *mmio = host->mmio;
  404. u32 msg = (u32) carm_ref_msg_dma(host, crq->tag);
  405. u32 cm_bucket = crq->msg_bucket;
  406. u32 tmp;
  407. int rc = 0;
  408. VPRINTK("ENTER\n");
  409. tmp = readl(mmio + CARM_HMUC);
  410. if (tmp & CARM_Q_FULL) {
  411. #if 0
  412. tmp = readl(mmio + CARM_INT_MASK);
  413. tmp |= INT_Q_AVAILABLE;
  414. writel(tmp, mmio + CARM_INT_MASK);
  415. readl(mmio + CARM_INT_MASK); /* flush */
  416. #endif
  417. DPRINTK("host msg queue full\n");
  418. rc = -EBUSY;
  419. } else {
  420. writel(msg | (cm_bucket << 1), mmio + CARM_IHQP);
  421. readl(mmio + CARM_IHQP); /* flush */
  422. }
  423. return rc;
  424. }
  425. static struct carm_request *carm_get_request(struct carm_host *host)
  426. {
  427. unsigned int i;
  428. /* obey global hardware limit on S/G entries */
  429. if (host->hw_sg_used >= (CARM_MAX_HOST_SG - CARM_MAX_REQ_SG))
  430. return NULL;
  431. for (i = 0; i < max_queue; i++)
  432. if ((host->msg_alloc & (1ULL << i)) == 0) {
  433. struct carm_request *crq = &host->req[i];
  434. crq->port = NULL;
  435. crq->n_elem = 0;
  436. host->msg_alloc |= (1ULL << i);
  437. host->n_msgs++;
  438. assert(host->n_msgs <= CARM_MAX_REQ);
  439. sg_init_table(crq->sg, CARM_MAX_REQ_SG);
  440. return crq;
  441. }
  442. DPRINTK("no request available, returning NULL\n");
  443. return NULL;
  444. }
  445. static int carm_put_request(struct carm_host *host, struct carm_request *crq)
  446. {
  447. assert(crq->tag < max_queue);
  448. if (unlikely((host->msg_alloc & (1ULL << crq->tag)) == 0))
  449. return -EINVAL; /* tried to clear a tag that was not active */
  450. assert(host->hw_sg_used >= crq->n_elem);
  451. host->msg_alloc &= ~(1ULL << crq->tag);
  452. host->hw_sg_used -= crq->n_elem;
  453. host->n_msgs--;
  454. return 0;
  455. }
  456. static struct carm_request *carm_get_special(struct carm_host *host)
  457. {
  458. unsigned long flags;
  459. struct carm_request *crq = NULL;
  460. struct request *rq;
  461. int tries = 5000;
  462. while (tries-- > 0) {
  463. spin_lock_irqsave(&host->lock, flags);
  464. crq = carm_get_request(host);
  465. spin_unlock_irqrestore(&host->lock, flags);
  466. if (crq)
  467. break;
  468. msleep(10);
  469. }
  470. if (!crq)
  471. return NULL;
  472. rq = blk_get_request(host->oob_q, WRITE /* bogus */, GFP_KERNEL);
  473. if (IS_ERR(rq)) {
  474. spin_lock_irqsave(&host->lock, flags);
  475. carm_put_request(host, crq);
  476. spin_unlock_irqrestore(&host->lock, flags);
  477. return NULL;
  478. }
  479. crq->rq = rq;
  480. return crq;
  481. }
  482. static int carm_array_info (struct carm_host *host, unsigned int array_idx)
  483. {
  484. struct carm_msg_ioctl *ioc;
  485. unsigned int idx;
  486. u32 msg_data;
  487. dma_addr_t msg_dma;
  488. struct carm_request *crq;
  489. int rc;
  490. crq = carm_get_special(host);
  491. if (!crq) {
  492. rc = -ENOMEM;
  493. goto err_out;
  494. }
  495. idx = crq->tag;
  496. ioc = carm_ref_msg(host, idx);
  497. msg_dma = carm_ref_msg_dma(host, idx);
  498. msg_data = (u32) (msg_dma + sizeof(struct carm_array_info));
  499. crq->msg_type = CARM_MSG_ARRAY;
  500. crq->msg_subtype = CARM_ARRAY_INFO;
  501. rc = carm_lookup_bucket(sizeof(struct carm_msg_ioctl) +
  502. sizeof(struct carm_array_info));
  503. BUG_ON(rc < 0);
  504. crq->msg_bucket = (u32) rc;
  505. memset(ioc, 0, sizeof(*ioc));
  506. ioc->type = CARM_MSG_ARRAY;
  507. ioc->subtype = CARM_ARRAY_INFO;
  508. ioc->array_id = (u8) array_idx;
  509. ioc->handle = cpu_to_le32(TAG_ENCODE(idx));
  510. ioc->data_addr = cpu_to_le32(msg_data);
  511. spin_lock_irq(&host->lock);
  512. assert(host->state == HST_DEV_SCAN_START ||
  513. host->state == HST_DEV_SCAN);
  514. spin_unlock_irq(&host->lock);
  515. DPRINTK("blk_execute_rq_nowait, tag == %u\n", idx);
  516. crq->rq->cmd_type = REQ_TYPE_DRV_PRIV;
  517. crq->rq->special = crq;
  518. blk_execute_rq_nowait(host->oob_q, NULL, crq->rq, true, NULL);
  519. return 0;
  520. err_out:
  521. spin_lock_irq(&host->lock);
  522. host->state = HST_ERROR;
  523. spin_unlock_irq(&host->lock);
  524. return rc;
  525. }
  526. typedef unsigned int (*carm_sspc_t)(struct carm_host *, unsigned int, void *);
  527. static int carm_send_special (struct carm_host *host, carm_sspc_t func)
  528. {
  529. struct carm_request *crq;
  530. struct carm_msg_ioctl *ioc;
  531. void *mem;
  532. unsigned int idx, msg_size;
  533. int rc;
  534. crq = carm_get_special(host);
  535. if (!crq)
  536. return -ENOMEM;
  537. idx = crq->tag;
  538. mem = carm_ref_msg(host, idx);
  539. msg_size = func(host, idx, mem);
  540. ioc = mem;
  541. crq->msg_type = ioc->type;
  542. crq->msg_subtype = ioc->subtype;
  543. rc = carm_lookup_bucket(msg_size);
  544. BUG_ON(rc < 0);
  545. crq->msg_bucket = (u32) rc;
  546. DPRINTK("blk_execute_rq_nowait, tag == %u\n", idx);
  547. crq->rq->cmd_type = REQ_TYPE_DRV_PRIV;
  548. crq->rq->special = crq;
  549. blk_execute_rq_nowait(host->oob_q, NULL, crq->rq, true, NULL);
  550. return 0;
  551. }
  552. static unsigned int carm_fill_sync_time(struct carm_host *host,
  553. unsigned int idx, void *mem)
  554. {
  555. struct carm_msg_sync_time *st = mem;
  556. time64_t tv = ktime_get_real_seconds();
  557. memset(st, 0, sizeof(*st));
  558. st->type = CARM_MSG_MISC;
  559. st->subtype = MISC_SET_TIME;
  560. st->handle = cpu_to_le32(TAG_ENCODE(idx));
  561. st->timestamp = cpu_to_le32(tv);
  562. return sizeof(struct carm_msg_sync_time);
  563. }
  564. static unsigned int carm_fill_alloc_buf(struct carm_host *host,
  565. unsigned int idx, void *mem)
  566. {
  567. struct carm_msg_allocbuf *ab = mem;
  568. memset(ab, 0, sizeof(*ab));
  569. ab->type = CARM_MSG_MISC;
  570. ab->subtype = MISC_ALLOC_MEM;
  571. ab->handle = cpu_to_le32(TAG_ENCODE(idx));
  572. ab->n_sg = 1;
  573. ab->sg_type = SGT_32BIT;
  574. ab->addr = cpu_to_le32(host->shm_dma + (PDC_SHM_SIZE >> 1));
  575. ab->len = cpu_to_le32(PDC_SHM_SIZE >> 1);
  576. ab->evt_pool = cpu_to_le32(host->shm_dma + (16 * 1024));
  577. ab->n_evt = cpu_to_le32(1024);
  578. ab->rbuf_pool = cpu_to_le32(host->shm_dma);
  579. ab->n_rbuf = cpu_to_le32(RMSG_Q_LEN);
  580. ab->msg_pool = cpu_to_le32(host->shm_dma + RBUF_LEN);
  581. ab->n_msg = cpu_to_le32(CARM_Q_LEN);
  582. ab->sg[0].start = cpu_to_le32(host->shm_dma + (PDC_SHM_SIZE >> 1));
  583. ab->sg[0].len = cpu_to_le32(65536);
  584. return sizeof(struct carm_msg_allocbuf);
  585. }
  586. static unsigned int carm_fill_scan_channels(struct carm_host *host,
  587. unsigned int idx, void *mem)
  588. {
  589. struct carm_msg_ioctl *ioc = mem;
  590. u32 msg_data = (u32) (carm_ref_msg_dma(host, idx) +
  591. IOC_SCAN_CHAN_OFFSET);
  592. memset(ioc, 0, sizeof(*ioc));
  593. ioc->type = CARM_MSG_IOCTL;
  594. ioc->subtype = CARM_IOC_SCAN_CHAN;
  595. ioc->handle = cpu_to_le32(TAG_ENCODE(idx));
  596. ioc->data_addr = cpu_to_le32(msg_data);
  597. /* fill output data area with "no device" default values */
  598. mem += IOC_SCAN_CHAN_OFFSET;
  599. memset(mem, IOC_SCAN_CHAN_NODEV, CARM_MAX_PORTS);
  600. return IOC_SCAN_CHAN_OFFSET + CARM_MAX_PORTS;
  601. }
  602. static unsigned int carm_fill_get_fw_ver(struct carm_host *host,
  603. unsigned int idx, void *mem)
  604. {
  605. struct carm_msg_get_fw_ver *ioc = mem;
  606. u32 msg_data = (u32) (carm_ref_msg_dma(host, idx) + sizeof(*ioc));
  607. memset(ioc, 0, sizeof(*ioc));
  608. ioc->type = CARM_MSG_MISC;
  609. ioc->subtype = MISC_GET_FW_VER;
  610. ioc->handle = cpu_to_le32(TAG_ENCODE(idx));
  611. ioc->data_addr = cpu_to_le32(msg_data);
  612. return sizeof(struct carm_msg_get_fw_ver) +
  613. sizeof(struct carm_fw_ver);
  614. }
  615. static inline void carm_end_request_queued(struct carm_host *host,
  616. struct carm_request *crq,
  617. int error)
  618. {
  619. struct request *req = crq->rq;
  620. int rc;
  621. __blk_end_request_all(req, error);
  622. rc = carm_put_request(host, crq);
  623. assert(rc == 0);
  624. }
  625. static inline void carm_push_q (struct carm_host *host, struct request_queue *q)
  626. {
  627. unsigned int idx = host->wait_q_prod % CARM_MAX_WAIT_Q;
  628. blk_stop_queue(q);
  629. VPRINTK("STOPPED QUEUE %p\n", q);
  630. host->wait_q[idx] = q;
  631. host->wait_q_prod++;
  632. BUG_ON(host->wait_q_prod == host->wait_q_cons); /* overrun */
  633. }
  634. static inline struct request_queue *carm_pop_q(struct carm_host *host)
  635. {
  636. unsigned int idx;
  637. if (host->wait_q_prod == host->wait_q_cons)
  638. return NULL;
  639. idx = host->wait_q_cons % CARM_MAX_WAIT_Q;
  640. host->wait_q_cons++;
  641. return host->wait_q[idx];
  642. }
  643. static inline void carm_round_robin(struct carm_host *host)
  644. {
  645. struct request_queue *q = carm_pop_q(host);
  646. if (q) {
  647. blk_start_queue(q);
  648. VPRINTK("STARTED QUEUE %p\n", q);
  649. }
  650. }
  651. static inline void carm_end_rq(struct carm_host *host, struct carm_request *crq,
  652. int error)
  653. {
  654. carm_end_request_queued(host, crq, error);
  655. if (max_queue == 1)
  656. carm_round_robin(host);
  657. else if ((host->n_msgs <= CARM_MSG_LOW_WATER) &&
  658. (host->hw_sg_used <= CARM_SG_LOW_WATER)) {
  659. carm_round_robin(host);
  660. }
  661. }
  662. static void carm_oob_rq_fn(struct request_queue *q)
  663. {
  664. struct carm_host *host = q->queuedata;
  665. struct carm_request *crq;
  666. struct request *rq;
  667. int rc;
  668. while (1) {
  669. DPRINTK("get req\n");
  670. rq = blk_fetch_request(q);
  671. if (!rq)
  672. break;
  673. crq = rq->special;
  674. assert(crq != NULL);
  675. assert(crq->rq == rq);
  676. crq->n_elem = 0;
  677. DPRINTK("send req\n");
  678. rc = carm_send_msg(host, crq);
  679. if (rc) {
  680. blk_requeue_request(q, rq);
  681. carm_push_q(host, q);
  682. return; /* call us again later, eventually */
  683. }
  684. }
  685. }
  686. static void carm_rq_fn(struct request_queue *q)
  687. {
  688. struct carm_port *port = q->queuedata;
  689. struct carm_host *host = port->host;
  690. struct carm_msg_rw *msg;
  691. struct carm_request *crq;
  692. struct request *rq;
  693. struct scatterlist *sg;
  694. int writing = 0, pci_dir, i, n_elem, rc;
  695. u32 tmp;
  696. unsigned int msg_size;
  697. queue_one_request:
  698. VPRINTK("get req\n");
  699. rq = blk_peek_request(q);
  700. if (!rq)
  701. return;
  702. crq = carm_get_request(host);
  703. if (!crq) {
  704. carm_push_q(host, q);
  705. return; /* call us again later, eventually */
  706. }
  707. crq->rq = rq;
  708. blk_start_request(rq);
  709. if (rq_data_dir(rq) == WRITE) {
  710. writing = 1;
  711. pci_dir = PCI_DMA_TODEVICE;
  712. } else {
  713. pci_dir = PCI_DMA_FROMDEVICE;
  714. }
  715. /* get scatterlist from block layer */
  716. sg = &crq->sg[0];
  717. n_elem = blk_rq_map_sg(q, rq, sg);
  718. if (n_elem <= 0) {
  719. carm_end_rq(host, crq, -EIO);
  720. return; /* request with no s/g entries? */
  721. }
  722. /* map scatterlist to PCI bus addresses */
  723. n_elem = pci_map_sg(host->pdev, sg, n_elem, pci_dir);
  724. if (n_elem <= 0) {
  725. carm_end_rq(host, crq, -EIO);
  726. return; /* request with no s/g entries? */
  727. }
  728. crq->n_elem = n_elem;
  729. crq->port = port;
  730. host->hw_sg_used += n_elem;
  731. /*
  732. * build read/write message
  733. */
  734. VPRINTK("build msg\n");
  735. msg = (struct carm_msg_rw *) carm_ref_msg(host, crq->tag);
  736. if (writing) {
  737. msg->type = CARM_MSG_WRITE;
  738. crq->msg_type = CARM_MSG_WRITE;
  739. } else {
  740. msg->type = CARM_MSG_READ;
  741. crq->msg_type = CARM_MSG_READ;
  742. }
  743. msg->id = port->port_no;
  744. msg->sg_count = n_elem;
  745. msg->sg_type = SGT_32BIT;
  746. msg->handle = cpu_to_le32(TAG_ENCODE(crq->tag));
  747. msg->lba = cpu_to_le32(blk_rq_pos(rq) & 0xffffffff);
  748. tmp = (blk_rq_pos(rq) >> 16) >> 16;
  749. msg->lba_high = cpu_to_le16( (u16) tmp );
  750. msg->lba_count = cpu_to_le16(blk_rq_sectors(rq));
  751. msg_size = sizeof(struct carm_msg_rw) - sizeof(msg->sg);
  752. for (i = 0; i < n_elem; i++) {
  753. struct carm_msg_sg *carm_sg = &msg->sg[i];
  754. carm_sg->start = cpu_to_le32(sg_dma_address(&crq->sg[i]));
  755. carm_sg->len = cpu_to_le32(sg_dma_len(&crq->sg[i]));
  756. msg_size += sizeof(struct carm_msg_sg);
  757. }
  758. rc = carm_lookup_bucket(msg_size);
  759. BUG_ON(rc < 0);
  760. crq->msg_bucket = (u32) rc;
  761. /*
  762. * queue read/write message to hardware
  763. */
  764. VPRINTK("send msg, tag == %u\n", crq->tag);
  765. rc = carm_send_msg(host, crq);
  766. if (rc) {
  767. carm_put_request(host, crq);
  768. blk_requeue_request(q, rq);
  769. carm_push_q(host, q);
  770. return; /* call us again later, eventually */
  771. }
  772. goto queue_one_request;
  773. }
  774. static void carm_handle_array_info(struct carm_host *host,
  775. struct carm_request *crq, u8 *mem,
  776. int error)
  777. {
  778. struct carm_port *port;
  779. u8 *msg_data = mem + sizeof(struct carm_array_info);
  780. struct carm_array_info *desc = (struct carm_array_info *) msg_data;
  781. u64 lo, hi;
  782. int cur_port;
  783. size_t slen;
  784. DPRINTK("ENTER\n");
  785. carm_end_rq(host, crq, error);
  786. if (error)
  787. goto out;
  788. if (le32_to_cpu(desc->array_status) & ARRAY_NO_EXIST)
  789. goto out;
  790. cur_port = host->cur_scan_dev;
  791. /* should never occur */
  792. if ((cur_port < 0) || (cur_port >= CARM_MAX_PORTS)) {
  793. printk(KERN_ERR PFX "BUG: cur_scan_dev==%d, array_id==%d\n",
  794. cur_port, (int) desc->array_id);
  795. goto out;
  796. }
  797. port = &host->port[cur_port];
  798. lo = (u64) le32_to_cpu(desc->size);
  799. hi = (u64) le16_to_cpu(desc->size_hi);
  800. port->capacity = lo | (hi << 32);
  801. port->dev_geom_head = le16_to_cpu(desc->head);
  802. port->dev_geom_sect = le16_to_cpu(desc->sect);
  803. port->dev_geom_cyl = le16_to_cpu(desc->cyl);
  804. host->dev_active |= (1 << cur_port);
  805. strncpy(port->name, desc->name, sizeof(port->name));
  806. port->name[sizeof(port->name) - 1] = 0;
  807. slen = strlen(port->name);
  808. while (slen && (port->name[slen - 1] == ' ')) {
  809. port->name[slen - 1] = 0;
  810. slen--;
  811. }
  812. printk(KERN_INFO DRV_NAME "(%s): port %u device %Lu sectors\n",
  813. pci_name(host->pdev), port->port_no,
  814. (unsigned long long) port->capacity);
  815. printk(KERN_INFO DRV_NAME "(%s): port %u device \"%s\"\n",
  816. pci_name(host->pdev), port->port_no, port->name);
  817. out:
  818. assert(host->state == HST_DEV_SCAN);
  819. schedule_work(&host->fsm_task);
  820. }
  821. static void carm_handle_scan_chan(struct carm_host *host,
  822. struct carm_request *crq, u8 *mem,
  823. int error)
  824. {
  825. u8 *msg_data = mem + IOC_SCAN_CHAN_OFFSET;
  826. unsigned int i, dev_count = 0;
  827. int new_state = HST_DEV_SCAN_START;
  828. DPRINTK("ENTER\n");
  829. carm_end_rq(host, crq, error);
  830. if (error) {
  831. new_state = HST_ERROR;
  832. goto out;
  833. }
  834. /* TODO: scan and support non-disk devices */
  835. for (i = 0; i < 8; i++)
  836. if (msg_data[i] == 0) { /* direct-access device (disk) */
  837. host->dev_present |= (1 << i);
  838. dev_count++;
  839. }
  840. printk(KERN_INFO DRV_NAME "(%s): found %u interesting devices\n",
  841. pci_name(host->pdev), dev_count);
  842. out:
  843. assert(host->state == HST_PORT_SCAN);
  844. host->state = new_state;
  845. schedule_work(&host->fsm_task);
  846. }
  847. static void carm_handle_generic(struct carm_host *host,
  848. struct carm_request *crq, int error,
  849. int cur_state, int next_state)
  850. {
  851. DPRINTK("ENTER\n");
  852. carm_end_rq(host, crq, error);
  853. assert(host->state == cur_state);
  854. if (error)
  855. host->state = HST_ERROR;
  856. else
  857. host->state = next_state;
  858. schedule_work(&host->fsm_task);
  859. }
  860. static inline void carm_handle_rw(struct carm_host *host,
  861. struct carm_request *crq, int error)
  862. {
  863. int pci_dir;
  864. VPRINTK("ENTER\n");
  865. if (rq_data_dir(crq->rq) == WRITE)
  866. pci_dir = PCI_DMA_TODEVICE;
  867. else
  868. pci_dir = PCI_DMA_FROMDEVICE;
  869. pci_unmap_sg(host->pdev, &crq->sg[0], crq->n_elem, pci_dir);
  870. carm_end_rq(host, crq, error);
  871. }
  872. static inline void carm_handle_resp(struct carm_host *host,
  873. __le32 ret_handle_le, u32 status)
  874. {
  875. u32 handle = le32_to_cpu(ret_handle_le);
  876. unsigned int msg_idx;
  877. struct carm_request *crq;
  878. int error = (status == RMSG_OK) ? 0 : -EIO;
  879. u8 *mem;
  880. VPRINTK("ENTER, handle == 0x%x\n", handle);
  881. if (unlikely(!TAG_VALID(handle))) {
  882. printk(KERN_ERR DRV_NAME "(%s): BUG: invalid tag 0x%x\n",
  883. pci_name(host->pdev), handle);
  884. return;
  885. }
  886. msg_idx = TAG_DECODE(handle);
  887. VPRINTK("tag == %u\n", msg_idx);
  888. crq = &host->req[msg_idx];
  889. /* fast path */
  890. if (likely(crq->msg_type == CARM_MSG_READ ||
  891. crq->msg_type == CARM_MSG_WRITE)) {
  892. carm_handle_rw(host, crq, error);
  893. return;
  894. }
  895. mem = carm_ref_msg(host, msg_idx);
  896. switch (crq->msg_type) {
  897. case CARM_MSG_IOCTL: {
  898. switch (crq->msg_subtype) {
  899. case CARM_IOC_SCAN_CHAN:
  900. carm_handle_scan_chan(host, crq, mem, error);
  901. break;
  902. default:
  903. /* unknown / invalid response */
  904. goto err_out;
  905. }
  906. break;
  907. }
  908. case CARM_MSG_MISC: {
  909. switch (crq->msg_subtype) {
  910. case MISC_ALLOC_MEM:
  911. carm_handle_generic(host, crq, error,
  912. HST_ALLOC_BUF, HST_SYNC_TIME);
  913. break;
  914. case MISC_SET_TIME:
  915. carm_handle_generic(host, crq, error,
  916. HST_SYNC_TIME, HST_GET_FW_VER);
  917. break;
  918. case MISC_GET_FW_VER: {
  919. struct carm_fw_ver *ver = (struct carm_fw_ver *)
  920. (mem + sizeof(struct carm_msg_get_fw_ver));
  921. if (!error) {
  922. host->fw_ver = le32_to_cpu(ver->version);
  923. host->flags |= (ver->features & FL_FW_VER_MASK);
  924. }
  925. carm_handle_generic(host, crq, error,
  926. HST_GET_FW_VER, HST_PORT_SCAN);
  927. break;
  928. }
  929. default:
  930. /* unknown / invalid response */
  931. goto err_out;
  932. }
  933. break;
  934. }
  935. case CARM_MSG_ARRAY: {
  936. switch (crq->msg_subtype) {
  937. case CARM_ARRAY_INFO:
  938. carm_handle_array_info(host, crq, mem, error);
  939. break;
  940. default:
  941. /* unknown / invalid response */
  942. goto err_out;
  943. }
  944. break;
  945. }
  946. default:
  947. /* unknown / invalid response */
  948. goto err_out;
  949. }
  950. return;
  951. err_out:
  952. printk(KERN_WARNING DRV_NAME "(%s): BUG: unhandled message type %d/%d\n",
  953. pci_name(host->pdev), crq->msg_type, crq->msg_subtype);
  954. carm_end_rq(host, crq, -EIO);
  955. }
  956. static inline void carm_handle_responses(struct carm_host *host)
  957. {
  958. void __iomem *mmio = host->mmio;
  959. struct carm_response *resp = (struct carm_response *) host->shm;
  960. unsigned int work = 0;
  961. unsigned int idx = host->resp_idx % RMSG_Q_LEN;
  962. while (1) {
  963. u32 status = le32_to_cpu(resp[idx].status);
  964. if (status == 0xffffffff) {
  965. VPRINTK("ending response on index %u\n", idx);
  966. writel(idx << 3, mmio + CARM_RESP_IDX);
  967. break;
  968. }
  969. /* response to a message we sent */
  970. else if ((status & (1 << 31)) == 0) {
  971. VPRINTK("handling msg response on index %u\n", idx);
  972. carm_handle_resp(host, resp[idx].ret_handle, status);
  973. resp[idx].status = cpu_to_le32(0xffffffff);
  974. }
  975. /* asynchronous events the hardware throws our way */
  976. else if ((status & 0xff000000) == (1 << 31)) {
  977. u8 *evt_type_ptr = (u8 *) &resp[idx];
  978. u8 evt_type = *evt_type_ptr;
  979. printk(KERN_WARNING DRV_NAME "(%s): unhandled event type %d\n",
  980. pci_name(host->pdev), (int) evt_type);
  981. resp[idx].status = cpu_to_le32(0xffffffff);
  982. }
  983. idx = NEXT_RESP(idx);
  984. work++;
  985. }
  986. VPRINTK("EXIT, work==%u\n", work);
  987. host->resp_idx += work;
  988. }
  989. static irqreturn_t carm_interrupt(int irq, void *__host)
  990. {
  991. struct carm_host *host = __host;
  992. void __iomem *mmio;
  993. u32 mask;
  994. int handled = 0;
  995. unsigned long flags;
  996. if (!host) {
  997. VPRINTK("no host\n");
  998. return IRQ_NONE;
  999. }
  1000. spin_lock_irqsave(&host->lock, flags);
  1001. mmio = host->mmio;
  1002. /* reading should also clear interrupts */
  1003. mask = readl(mmio + CARM_INT_STAT);
  1004. if (mask == 0 || mask == 0xffffffff) {
  1005. VPRINTK("no work, mask == 0x%x\n", mask);
  1006. goto out;
  1007. }
  1008. if (mask & INT_ACK_MASK)
  1009. writel(mask, mmio + CARM_INT_STAT);
  1010. if (unlikely(host->state == HST_INVALID)) {
  1011. VPRINTK("not initialized yet, mask = 0x%x\n", mask);
  1012. goto out;
  1013. }
  1014. if (mask & CARM_HAVE_RESP) {
  1015. handled = 1;
  1016. carm_handle_responses(host);
  1017. }
  1018. out:
  1019. spin_unlock_irqrestore(&host->lock, flags);
  1020. VPRINTK("EXIT\n");
  1021. return IRQ_RETVAL(handled);
  1022. }
  1023. static void carm_fsm_task (struct work_struct *work)
  1024. {
  1025. struct carm_host *host =
  1026. container_of(work, struct carm_host, fsm_task);
  1027. unsigned long flags;
  1028. unsigned int state;
  1029. int rc, i, next_dev;
  1030. int reschedule = 0;
  1031. int new_state = HST_INVALID;
  1032. spin_lock_irqsave(&host->lock, flags);
  1033. state = host->state;
  1034. spin_unlock_irqrestore(&host->lock, flags);
  1035. DPRINTK("ENTER, state == %s\n", state_name[state]);
  1036. switch (state) {
  1037. case HST_PROBE_START:
  1038. new_state = HST_ALLOC_BUF;
  1039. reschedule = 1;
  1040. break;
  1041. case HST_ALLOC_BUF:
  1042. rc = carm_send_special(host, carm_fill_alloc_buf);
  1043. if (rc) {
  1044. new_state = HST_ERROR;
  1045. reschedule = 1;
  1046. }
  1047. break;
  1048. case HST_SYNC_TIME:
  1049. rc = carm_send_special(host, carm_fill_sync_time);
  1050. if (rc) {
  1051. new_state = HST_ERROR;
  1052. reschedule = 1;
  1053. }
  1054. break;
  1055. case HST_GET_FW_VER:
  1056. rc = carm_send_special(host, carm_fill_get_fw_ver);
  1057. if (rc) {
  1058. new_state = HST_ERROR;
  1059. reschedule = 1;
  1060. }
  1061. break;
  1062. case HST_PORT_SCAN:
  1063. rc = carm_send_special(host, carm_fill_scan_channels);
  1064. if (rc) {
  1065. new_state = HST_ERROR;
  1066. reschedule = 1;
  1067. }
  1068. break;
  1069. case HST_DEV_SCAN_START:
  1070. host->cur_scan_dev = -1;
  1071. new_state = HST_DEV_SCAN;
  1072. reschedule = 1;
  1073. break;
  1074. case HST_DEV_SCAN:
  1075. next_dev = -1;
  1076. for (i = host->cur_scan_dev + 1; i < CARM_MAX_PORTS; i++)
  1077. if (host->dev_present & (1 << i)) {
  1078. next_dev = i;
  1079. break;
  1080. }
  1081. if (next_dev >= 0) {
  1082. host->cur_scan_dev = next_dev;
  1083. rc = carm_array_info(host, next_dev);
  1084. if (rc) {
  1085. new_state = HST_ERROR;
  1086. reschedule = 1;
  1087. }
  1088. } else {
  1089. new_state = HST_DEV_ACTIVATE;
  1090. reschedule = 1;
  1091. }
  1092. break;
  1093. case HST_DEV_ACTIVATE: {
  1094. int activated = 0;
  1095. for (i = 0; i < CARM_MAX_PORTS; i++)
  1096. if (host->dev_active & (1 << i)) {
  1097. struct carm_port *port = &host->port[i];
  1098. struct gendisk *disk = port->disk;
  1099. set_capacity(disk, port->capacity);
  1100. add_disk(disk);
  1101. activated++;
  1102. }
  1103. printk(KERN_INFO DRV_NAME "(%s): %d ports activated\n",
  1104. pci_name(host->pdev), activated);
  1105. new_state = HST_PROBE_FINISHED;
  1106. reschedule = 1;
  1107. break;
  1108. }
  1109. case HST_PROBE_FINISHED:
  1110. complete(&host->probe_comp);
  1111. break;
  1112. case HST_ERROR:
  1113. /* FIXME: TODO */
  1114. break;
  1115. default:
  1116. /* should never occur */
  1117. printk(KERN_ERR PFX "BUG: unknown state %d\n", state);
  1118. assert(0);
  1119. break;
  1120. }
  1121. if (new_state != HST_INVALID) {
  1122. spin_lock_irqsave(&host->lock, flags);
  1123. host->state = new_state;
  1124. spin_unlock_irqrestore(&host->lock, flags);
  1125. }
  1126. if (reschedule)
  1127. schedule_work(&host->fsm_task);
  1128. }
  1129. static int carm_init_wait(void __iomem *mmio, u32 bits, unsigned int test_bit)
  1130. {
  1131. unsigned int i;
  1132. for (i = 0; i < 50000; i++) {
  1133. u32 tmp = readl(mmio + CARM_LMUC);
  1134. udelay(100);
  1135. if (test_bit) {
  1136. if ((tmp & bits) == bits)
  1137. return 0;
  1138. } else {
  1139. if ((tmp & bits) == 0)
  1140. return 0;
  1141. }
  1142. cond_resched();
  1143. }
  1144. printk(KERN_ERR PFX "carm_init_wait timeout, bits == 0x%x, test_bit == %s\n",
  1145. bits, test_bit ? "yes" : "no");
  1146. return -EBUSY;
  1147. }
  1148. static void carm_init_responses(struct carm_host *host)
  1149. {
  1150. void __iomem *mmio = host->mmio;
  1151. unsigned int i;
  1152. struct carm_response *resp = (struct carm_response *) host->shm;
  1153. for (i = 0; i < RMSG_Q_LEN; i++)
  1154. resp[i].status = cpu_to_le32(0xffffffff);
  1155. writel(0, mmio + CARM_RESP_IDX);
  1156. }
  1157. static int carm_init_host(struct carm_host *host)
  1158. {
  1159. void __iomem *mmio = host->mmio;
  1160. u32 tmp;
  1161. u8 tmp8;
  1162. int rc;
  1163. DPRINTK("ENTER\n");
  1164. writel(0, mmio + CARM_INT_MASK);
  1165. tmp8 = readb(mmio + CARM_INITC);
  1166. if (tmp8 & 0x01) {
  1167. tmp8 &= ~0x01;
  1168. writeb(tmp8, mmio + CARM_INITC);
  1169. readb(mmio + CARM_INITC); /* flush */
  1170. DPRINTK("snooze...\n");
  1171. msleep(5000);
  1172. }
  1173. tmp = readl(mmio + CARM_HMUC);
  1174. if (tmp & CARM_CME) {
  1175. DPRINTK("CME bit present, waiting\n");
  1176. rc = carm_init_wait(mmio, CARM_CME, 1);
  1177. if (rc) {
  1178. DPRINTK("EXIT, carm_init_wait 1 failed\n");
  1179. return rc;
  1180. }
  1181. }
  1182. if (tmp & CARM_RME) {
  1183. DPRINTK("RME bit present, waiting\n");
  1184. rc = carm_init_wait(mmio, CARM_RME, 1);
  1185. if (rc) {
  1186. DPRINTK("EXIT, carm_init_wait 2 failed\n");
  1187. return rc;
  1188. }
  1189. }
  1190. tmp &= ~(CARM_RME | CARM_CME);
  1191. writel(tmp, mmio + CARM_HMUC);
  1192. readl(mmio + CARM_HMUC); /* flush */
  1193. rc = carm_init_wait(mmio, CARM_RME | CARM_CME, 0);
  1194. if (rc) {
  1195. DPRINTK("EXIT, carm_init_wait 3 failed\n");
  1196. return rc;
  1197. }
  1198. carm_init_buckets(mmio);
  1199. writel(host->shm_dma & 0xffffffff, mmio + RBUF_ADDR_LO);
  1200. writel((host->shm_dma >> 16) >> 16, mmio + RBUF_ADDR_HI);
  1201. writel(RBUF_LEN, mmio + RBUF_BYTE_SZ);
  1202. tmp = readl(mmio + CARM_HMUC);
  1203. tmp |= (CARM_RME | CARM_CME | CARM_WZBC);
  1204. writel(tmp, mmio + CARM_HMUC);
  1205. readl(mmio + CARM_HMUC); /* flush */
  1206. rc = carm_init_wait(mmio, CARM_RME | CARM_CME, 1);
  1207. if (rc) {
  1208. DPRINTK("EXIT, carm_init_wait 4 failed\n");
  1209. return rc;
  1210. }
  1211. writel(0, mmio + CARM_HMPHA);
  1212. writel(INT_DEF_MASK, mmio + CARM_INT_MASK);
  1213. carm_init_responses(host);
  1214. /* start initialization, probing state machine */
  1215. spin_lock_irq(&host->lock);
  1216. assert(host->state == HST_INVALID);
  1217. host->state = HST_PROBE_START;
  1218. spin_unlock_irq(&host->lock);
  1219. schedule_work(&host->fsm_task);
  1220. DPRINTK("EXIT\n");
  1221. return 0;
  1222. }
  1223. static int carm_init_disks(struct carm_host *host)
  1224. {
  1225. unsigned int i;
  1226. int rc = 0;
  1227. for (i = 0; i < CARM_MAX_PORTS; i++) {
  1228. struct gendisk *disk;
  1229. struct request_queue *q;
  1230. struct carm_port *port;
  1231. port = &host->port[i];
  1232. port->host = host;
  1233. port->port_no = i;
  1234. disk = alloc_disk(CARM_MINORS_PER_MAJOR);
  1235. if (!disk) {
  1236. rc = -ENOMEM;
  1237. break;
  1238. }
  1239. port->disk = disk;
  1240. sprintf(disk->disk_name, DRV_NAME "/%u",
  1241. (unsigned int) (host->id * CARM_MAX_PORTS) + i);
  1242. disk->major = host->major;
  1243. disk->first_minor = i * CARM_MINORS_PER_MAJOR;
  1244. disk->fops = &carm_bd_ops;
  1245. disk->private_data = port;
  1246. q = blk_init_queue(carm_rq_fn, &host->lock);
  1247. if (!q) {
  1248. rc = -ENOMEM;
  1249. break;
  1250. }
  1251. disk->queue = q;
  1252. blk_queue_max_segments(q, CARM_MAX_REQ_SG);
  1253. blk_queue_segment_boundary(q, CARM_SG_BOUNDARY);
  1254. q->queuedata = port;
  1255. }
  1256. return rc;
  1257. }
  1258. static void carm_free_disks(struct carm_host *host)
  1259. {
  1260. unsigned int i;
  1261. for (i = 0; i < CARM_MAX_PORTS; i++) {
  1262. struct gendisk *disk = host->port[i].disk;
  1263. if (disk) {
  1264. struct request_queue *q = disk->queue;
  1265. if (disk->flags & GENHD_FL_UP)
  1266. del_gendisk(disk);
  1267. if (q)
  1268. blk_cleanup_queue(q);
  1269. put_disk(disk);
  1270. }
  1271. }
  1272. }
  1273. static int carm_init_shm(struct carm_host *host)
  1274. {
  1275. host->shm = pci_alloc_consistent(host->pdev, CARM_SHM_SIZE,
  1276. &host->shm_dma);
  1277. if (!host->shm)
  1278. return -ENOMEM;
  1279. host->msg_base = host->shm + RBUF_LEN;
  1280. host->msg_dma = host->shm_dma + RBUF_LEN;
  1281. memset(host->shm, 0xff, RBUF_LEN);
  1282. memset(host->msg_base, 0, PDC_SHM_SIZE - RBUF_LEN);
  1283. return 0;
  1284. }
  1285. static int carm_init_one (struct pci_dev *pdev, const struct pci_device_id *ent)
  1286. {
  1287. struct carm_host *host;
  1288. unsigned int pci_dac;
  1289. int rc;
  1290. struct request_queue *q;
  1291. unsigned int i;
  1292. printk_once(KERN_DEBUG DRV_NAME " version " DRV_VERSION "\n");
  1293. rc = pci_enable_device(pdev);
  1294. if (rc)
  1295. return rc;
  1296. rc = pci_request_regions(pdev, DRV_NAME);
  1297. if (rc)
  1298. goto err_out;
  1299. #ifdef IF_64BIT_DMA_IS_POSSIBLE /* grrrr... */
  1300. rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
  1301. if (!rc) {
  1302. rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
  1303. if (rc) {
  1304. printk(KERN_ERR DRV_NAME "(%s): consistent DMA mask failure\n",
  1305. pci_name(pdev));
  1306. goto err_out_regions;
  1307. }
  1308. pci_dac = 1;
  1309. } else {
  1310. #endif
  1311. rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
  1312. if (rc) {
  1313. printk(KERN_ERR DRV_NAME "(%s): DMA mask failure\n",
  1314. pci_name(pdev));
  1315. goto err_out_regions;
  1316. }
  1317. pci_dac = 0;
  1318. #ifdef IF_64BIT_DMA_IS_POSSIBLE /* grrrr... */
  1319. }
  1320. #endif
  1321. host = kzalloc(sizeof(*host), GFP_KERNEL);
  1322. if (!host) {
  1323. printk(KERN_ERR DRV_NAME "(%s): memory alloc failure\n",
  1324. pci_name(pdev));
  1325. rc = -ENOMEM;
  1326. goto err_out_regions;
  1327. }
  1328. host->pdev = pdev;
  1329. host->flags = pci_dac ? FL_DAC : 0;
  1330. spin_lock_init(&host->lock);
  1331. INIT_WORK(&host->fsm_task, carm_fsm_task);
  1332. init_completion(&host->probe_comp);
  1333. for (i = 0; i < ARRAY_SIZE(host->req); i++)
  1334. host->req[i].tag = i;
  1335. host->mmio = ioremap(pci_resource_start(pdev, 0),
  1336. pci_resource_len(pdev, 0));
  1337. if (!host->mmio) {
  1338. printk(KERN_ERR DRV_NAME "(%s): MMIO alloc failure\n",
  1339. pci_name(pdev));
  1340. rc = -ENOMEM;
  1341. goto err_out_kfree;
  1342. }
  1343. rc = carm_init_shm(host);
  1344. if (rc) {
  1345. printk(KERN_ERR DRV_NAME "(%s): DMA SHM alloc failure\n",
  1346. pci_name(pdev));
  1347. goto err_out_iounmap;
  1348. }
  1349. q = blk_init_queue(carm_oob_rq_fn, &host->lock);
  1350. if (!q) {
  1351. printk(KERN_ERR DRV_NAME "(%s): OOB queue alloc failure\n",
  1352. pci_name(pdev));
  1353. rc = -ENOMEM;
  1354. goto err_out_pci_free;
  1355. }
  1356. host->oob_q = q;
  1357. q->queuedata = host;
  1358. /*
  1359. * Figure out which major to use: 160, 161, or dynamic
  1360. */
  1361. if (!test_and_set_bit(0, &carm_major_alloc))
  1362. host->major = 160;
  1363. else if (!test_and_set_bit(1, &carm_major_alloc))
  1364. host->major = 161;
  1365. else
  1366. host->flags |= FL_DYN_MAJOR;
  1367. host->id = carm_host_id;
  1368. sprintf(host->name, DRV_NAME "%d", carm_host_id);
  1369. rc = register_blkdev(host->major, host->name);
  1370. if (rc < 0)
  1371. goto err_out_free_majors;
  1372. if (host->flags & FL_DYN_MAJOR)
  1373. host->major = rc;
  1374. rc = carm_init_disks(host);
  1375. if (rc)
  1376. goto err_out_blkdev_disks;
  1377. pci_set_master(pdev);
  1378. rc = request_irq(pdev->irq, carm_interrupt, IRQF_SHARED, DRV_NAME, host);
  1379. if (rc) {
  1380. printk(KERN_ERR DRV_NAME "(%s): irq alloc failure\n",
  1381. pci_name(pdev));
  1382. goto err_out_blkdev_disks;
  1383. }
  1384. rc = carm_init_host(host);
  1385. if (rc)
  1386. goto err_out_free_irq;
  1387. DPRINTK("waiting for probe_comp\n");
  1388. wait_for_completion(&host->probe_comp);
  1389. printk(KERN_INFO "%s: pci %s, ports %d, io %llx, irq %u, major %d\n",
  1390. host->name, pci_name(pdev), (int) CARM_MAX_PORTS,
  1391. (unsigned long long)pci_resource_start(pdev, 0),
  1392. pdev->irq, host->major);
  1393. carm_host_id++;
  1394. pci_set_drvdata(pdev, host);
  1395. return 0;
  1396. err_out_free_irq:
  1397. free_irq(pdev->irq, host);
  1398. err_out_blkdev_disks:
  1399. carm_free_disks(host);
  1400. unregister_blkdev(host->major, host->name);
  1401. err_out_free_majors:
  1402. if (host->major == 160)
  1403. clear_bit(0, &carm_major_alloc);
  1404. else if (host->major == 161)
  1405. clear_bit(1, &carm_major_alloc);
  1406. blk_cleanup_queue(host->oob_q);
  1407. err_out_pci_free:
  1408. pci_free_consistent(pdev, CARM_SHM_SIZE, host->shm, host->shm_dma);
  1409. err_out_iounmap:
  1410. iounmap(host->mmio);
  1411. err_out_kfree:
  1412. kfree(host);
  1413. err_out_regions:
  1414. pci_release_regions(pdev);
  1415. err_out:
  1416. pci_disable_device(pdev);
  1417. return rc;
  1418. }
  1419. static void carm_remove_one (struct pci_dev *pdev)
  1420. {
  1421. struct carm_host *host = pci_get_drvdata(pdev);
  1422. if (!host) {
  1423. printk(KERN_ERR PFX "BUG: no host data for PCI(%s)\n",
  1424. pci_name(pdev));
  1425. return;
  1426. }
  1427. free_irq(pdev->irq, host);
  1428. carm_free_disks(host);
  1429. unregister_blkdev(host->major, host->name);
  1430. if (host->major == 160)
  1431. clear_bit(0, &carm_major_alloc);
  1432. else if (host->major == 161)
  1433. clear_bit(1, &carm_major_alloc);
  1434. blk_cleanup_queue(host->oob_q);
  1435. pci_free_consistent(pdev, CARM_SHM_SIZE, host->shm, host->shm_dma);
  1436. iounmap(host->mmio);
  1437. kfree(host);
  1438. pci_release_regions(pdev);
  1439. pci_disable_device(pdev);
  1440. }
  1441. module_pci_driver(carm_driver);